Review Board 1.7.22


Review request for SQOOP-1056 "Implement connection resiliency in Sqoop using pluggable failure handlers"

Review Request #11320 - Created May 21, 2013 and updated

Shuaishuai Nie
trunk
SQOOP-1056
Reviewers
Sqoop
venkatnrangan
sqoop-trunk
Implement a pluggable way for handling connection failures, and/or intermittent errors in Sqoop. This is especially crucial in environments where the probability of connections getting reset or throttled is high.
In case of intermittent failures in Sqoop, due to connection losses or server throttling, Sqoop does not recover from those failures. As a result, the running Sqoop task would eventually fail, and a new task is started. In those cases, Sqoop does not always guarantee that tasks can safely be restarted. For example, if part of the records is already committed to the database, then restarting the task would result in some failures like primary key violations. Even for Sqoop jobs which commit the records only at the end of the task, any failures towards the end of the task would involve reprocessing the whole range of split owned by the task, and any progress is lost. (More detail please refer to the document attached at SQOOP-1056)
Tested with fault injection on SQL server integration tests from SQOOP-1035
build.xml
Revision 74150e6 New Change
[20] 190 lines
[+20]
191
  <property name="build.classes" location="${build.dir}/classes"/>
191
  <property name="build.classes" location="${build.dir}/classes"/>
192

    
   
192

   
193
  <!-- root directory for output/intermediate data for testing -->
193
  <!-- root directory for output/intermediate data for testing -->
194
  <property name="build.test" location="${build.dir}/test"/>
194
  <property name="build.test" location="${build.dir}/test"/>
195
  <property name="test.log.dir" location="${build.dir}/test/logs"/>
195
  <property name="test.log.dir" location="${build.dir}/test/logs"/>

    
   
196
  <property name="test.build.extraconf" value="${build.test}/extraconf" />
196

    
   
197

   
197
  <!-- compiled test classes -->
198
  <!-- compiled test classes -->
198
  <property name="build.test.classes" location="${build.test}/classes" />
199
  <property name="build.test.classes" location="${build.test}/classes" />
199

    
   
200

   
200
  <!-- compiled "perftest" programs -->
201
  <!-- compiled "perftest" programs -->
[+20] [20] 52 lines
[+20]
253
  <property name="cobertura.home" value="${cobertura.dir}" />
254
  <property name="cobertura.home" value="${cobertura.dir}" />
254
  <property name="cobertura.report.dir" value="${cobertura.dir}/report" />
255
  <property name="cobertura.report.dir" value="${cobertura.dir}/report" />
255
  <property name="cobertura.format" value="html" /> <!-- may be 'xml' -->
256
  <property name="cobertura.format" value="html" /> <!-- may be 'xml' -->
256
  <property name="cobertura.class.dir" value="${cobertura.dir}/classes" />
257
  <property name="cobertura.class.dir" value="${cobertura.dir}/classes" />
257

    
   
258

   

    
   
259
  <!-- aspectJ fault injection -->

    
   
260
  <import file="${test.dir}/aop/build/aop.xml"/>
258
  <!-- Checking code style -->
261
  <!-- Checking code style -->
259
  <property name="checkstyle.xml" value="${test.dir}/checkstyle.xml" />
262
  <property name="checkstyle.xml" value="${test.dir}/checkstyle.xml" />
260
  <property name="checkstyle.format.xsl"
263
  <property name="checkstyle.format.xsl"
261
      value="${test.dir}/checkstyle-noframes.xsl" />
264
      value="${test.dir}/checkstyle-noframes.xsl" />
262
  <property name="checkstyle.report.dir" value="${build.dir}" />
265
  <property name="checkstyle.report.dir" value="${build.dir}" />
[+20] [20] 149 lines
[+20]
412
    </path>
415
    </path>
413

    
   
416

   
414
    <!-- Classpath for unit tests (superset of compile.classpath) -->
417
    <!-- Classpath for unit tests (superset of compile.classpath) -->
415
    <path id="test.classpath">
418
    <path id="test.classpath">
416
      <pathelement location="${build.test.classes}" />
419
      <pathelement location="${build.test.classes}" />

    
   
420
      <pathelement location="${test.build.extraconf}"/>
417
      <path refid="${name}.hadooptest.classpath" />
421
      <path refid="${name}.hadooptest.classpath" />
418
      <path refid="compile.classpath" />
422
      <path refid="compile.classpath" />
419
    </path>
423
    </path>
420
  </target>
424
  </target>
421

    
   
425

   
[+20] [20] 56 lines
[+20]
478

    
   
482

   
479
  <target name="compile-test"
483
  <target name="compile-test"
480
      depends="compile, ivy-retrieve-hadoop-test"
484
      depends="compile, ivy-retrieve-hadoop-test"
481
      description="Compile test classes">
485
      description="Compile test classes">
482
    <mkdir dir="${build.test.classes}" />
486
    <mkdir dir="${build.test.classes}" />

    
   
487
    <mkdir dir="${test.build.extraconf}"/>
483
    <javac
488
    <javac
484
        encoding="${build.encoding}"
489
        encoding="${build.encoding}"
485
        srcdir="${test.dir}"
490
        srcdir="${test.dir}"
486
        includes="**/*.java"
491
        includes="**/*.java"
487
        destdir="${build.test.classes}"
492
        destdir="${build.test.classes}"
[+20] [20] 315 lines
[+20]
803
    <delete dir="${test.log.dir}"/>
808
    <delete dir="${test.log.dir}"/>
804
    <mkdir dir="${test.log.dir}"/>
809
    <mkdir dir="${test.log.dir}"/>
805
    <delete dir="${build.test}/data"/>
810
    <delete dir="${build.test}/data"/>
806
    <mkdir dir="${build.test}/data/sqoop" />
811
    <mkdir dir="${build.test}/data/sqoop" />
807
    <mkdir dir="${cobertura.class.dir}" />
812
    <mkdir dir="${cobertura.class.dir}" />

    
   
813
    <copy file="${test.dir}/fi-site.xml"

    
   
814
            todir="${test.build.extraconf}" />
808
    <junit
815
    <junit
809
      printsummary="yes" showoutput="${test.output}"
816
      printsummary="yes" showoutput="${test.output}"
810
      haltonfailure="no" fork="yes" maxmemory="512m"
817
      haltonfailure="no" fork="yes" maxmemory="512m"
811
      errorProperty="tests.failed" failureProperty="tests.failed"
818
      errorProperty="tests.failed" failureProperty="tests.failed"
812
      timeout="${test.timeout}"
819
      timeout="${test.timeout}"
[+20] [20] 589 lines
[+20]
1402
        <output path="${build.dir}/eclipse-build" />
1409
        <output path="${build.dir}/eclipse-build" />
1403
        <library pathref="test.classpath" />
1410
        <library pathref="test.classpath" />
1404
      </classpath>
1411
      </classpath>
1405
    </eclipse>
1412
    </eclipse>
1406
  </target>
1413
  </target>

    
   
1414

   

    
   
1415
  <!-- Fault injection customization section -->

    
   
1416
  <target name="justjar" depends="ivy-resolve-test">

    
   
1417
    <echo message="Project: ${ant.project.name}" />

    
   
1418
    <jar jarfile="${build.dir}/${test.jar}" basedir="${build.test.classes}"/>

    
   
1419
  </target>

    
   
1420

   

    
   
1421
  <target name="jar-fault-inject" depends="init, injectfaults"

    
   
1422
    description="Make sqoop-fi.jar">

    
   
1423
    <macro-jar-fault-inject

    
   
1424
      target.name="justjar"

    
   
1425
      build.dir="${build-fi.dir}"

    
   
1426
      jar.final.name="final.name"

    
   
1427
      jar.final.value="${final.name}-fi" />

    
   
1428
    <copy todir="${lib.dir}">

    
   
1429
      <fileset dir="${build-fi.dir}">

    
   
1430
        <include name="*.jar"/>

    
   
1431
      </fileset>

    
   
1432
    </copy>

    
   
1433
  </target>

    
   
1434

   

    
   
1435
  <target name="jar-test-fault-inject" depends="init, injectfaults"

    
   
1436
    description="Make sqoop-test-fi.jar">

    
   
1437
    <macro-jar-test-fault-inject

    
   
1438
      target.name="test-jar"

    
   
1439
      jar.final.name="test.final.name"

    
   
1440
      jar.final.value="${test.final.name}-fi" />

    
   
1441
  </target>

    
   
1442

   

    
   
1443
  <target name="run-fault-inject-with-testcaseonly" depends="init, injectfaults">

    
   
1444
    <fail unless="testcase">Can't run this target without -Dtestcase setting!

    
   
1445
    </fail>

    
   
1446
    <macro-run-tests-fault-inject target.name="test"

    
   
1447
      testcasesonly="true"/>

    
   
1448
  </target>

    
   
1449

   

    
   
1450
  <target name="run-test-core-fault-inject" depends="init, injectfaults"

    
   
1451
    description="Run full set of the unit tests with fault injection">

    
   
1452
    <macro-run-tests-fault-inject target.name="test"

    
   
1453
      testcasesonly="false"/>

    
   
1454
  </target>

    
   
1455

   
1407
</project>
1456
</project>
ivy.xml
Revision c00d30c New Change
 
ivy/libraries.properties
Revision 6b9b2d7 New Change
 
src/java/org/apache/sqoop/manager/SQLServerManager.java
Revision eea5a1d New Change
 
src/java/org/apache/sqoop/manager/SqlManager.java
Revision 8248263 New Change
 
src/java/org/apache/sqoop/mapreduce/SQLServerAsyncDBExecThread.java
New File
 
src/java/org/apache/sqoop/mapreduce/SQLServerExportDBExecThread.java
New File
 
src/java/org/apache/sqoop/mapreduce/SQLServerResilientExportOutputFormat.java
New File
 
src/java/org/apache/sqoop/mapreduce/SQLServerResilientUpdateOutputFormat.java
New File
 
src/java/org/apache/sqoop/mapreduce/SQLServerUpdateDBExecThread.java
New File
 
src/java/org/apache/sqoop/mapreduce/db/BasicRetrySQLFailureHandler.java
New File
 
src/java/org/apache/sqoop/mapreduce/db/DBRecordReader.java
Revision 7b9e4d2 New Change
 
src/java/org/apache/sqoop/mapreduce/db/DataDrivenDBRecordReader.java
Revision 4b44244 New Change
 
src/java/org/apache/sqoop/mapreduce/db/SQLFailureHandler.java
New File
 
src/java/org/apache/sqoop/mapreduce/db/SQLServerConnectionFailureHandler.java
New File
 
src/java/org/apache/sqoop/mapreduce/db/SQLServerDBInputFormat.java
New File
 
src/java/org/apache/sqoop/mapreduce/db/SQLServerDBRecordReader.java
New File
 
src/java/org/apache/sqoop/mapreduce/sqlserver/SqlServerRecordReader.java
Revision 2c08f12 New Change
 
src/test/fi-site.xml
New File
 
src/test/aop/build/aop.xml
New File
 
src/test/aop/org/apache/sqoop/fi/FiConfig.java
New File
 
src/test/aop/org/apache/sqoop/fi/ProbabilityModel.java
New File
 
src/test/aop/org/apache/sqoop/mapreduce/SqlServerExportAspect.aj
New File
 
src/test/aop/org/apache/sqoop/mapreduce/db/SqlServerImportAspect.aj
New File
 
  1. build.xml: Loading...
  2. ivy.xml: Loading...
  3. ivy/libraries.properties: Loading...
  4. src/java/org/apache/sqoop/manager/SQLServerManager.java: Loading...
  5. src/java/org/apache/sqoop/manager/SqlManager.java: Loading...
  6. src/java/org/apache/sqoop/mapreduce/SQLServerAsyncDBExecThread.java: Loading...
  7. src/java/org/apache/sqoop/mapreduce/SQLServerExportDBExecThread.java: Loading...
  8. src/java/org/apache/sqoop/mapreduce/SQLServerResilientExportOutputFormat.java: Loading...
  9. src/java/org/apache/sqoop/mapreduce/SQLServerResilientUpdateOutputFormat.java: Loading...
  10. src/java/org/apache/sqoop/mapreduce/SQLServerUpdateDBExecThread.java: Loading...
  11. src/java/org/apache/sqoop/mapreduce/db/BasicRetrySQLFailureHandler.java: Loading...
  12. src/java/org/apache/sqoop/mapreduce/db/DBRecordReader.java: Loading...
  13. src/java/org/apache/sqoop/mapreduce/db/DataDrivenDBRecordReader.java: Loading...
  14. src/java/org/apache/sqoop/mapreduce/db/SQLFailureHandler.java: Loading...
  15. src/java/org/apache/sqoop/mapreduce/db/SQLServerConnectionFailureHandler.java: Loading...
  16. src/java/org/apache/sqoop/mapreduce/db/SQLServerDBInputFormat.java: Loading...
  17. src/java/org/apache/sqoop/mapreduce/db/SQLServerDBRecordReader.java: Loading...
  18. src/java/org/apache/sqoop/mapreduce/sqlserver/SqlServerRecordReader.java: Loading...
  19. src/test/fi-site.xml: Loading...
  20. src/test/aop/build/aop.xml: Loading...
  21. src/test/aop/org/apache/sqoop/fi/FiConfig.java: Loading...
  22. src/test/aop/org/apache/sqoop/fi/ProbabilityModel.java: Loading...
  23. src/test/aop/org/apache/sqoop/mapreduce/SqlServerExportAspect.aj: Loading...
  24. src/test/aop/org/apache/sqoop/mapreduce/db/SqlServerImportAspect.aj: Loading...