Review Board 1.7.22


SQOOP-1172 Make Sqoop compatible with HBase 0.95+

Review Request #13524 - Created Aug. 13, 2013 and updated

Jarek Cecho
SQOOP-1172
Reviewers
Sqoop
sqoop-trunk
Tweaked the build system to support both HBase 0.94 and 0.95.
It seems that ant do not accepts "*" for test case names, so one need to execute them one by one:

1) Create list of HBase related test cases:
find . -iname "*HBase*Test.java"  | cut -d"/" -f 8 | sed -re "s/.java//" > hbase-tests

2) Execute each test case one-by-one

2.1) For HBase 0.95 on Hadoop 2.0.0:
for test in `cat hbase-tests`; do ant clean test -Dhadoopversion=200 -Dhbaseprofile=95 -Dhbasecompatprofile=2 -Dtestcase=$test ; done > log

2.2) For HBase 0.95 on Hadoop 1.0.0:
for test in `cat hbase-tests`; do ant clean test -Dhadoopversion=100 -Dhbaseprofile=95 -Dhbasecompatprofile=1 -Dtestcase=$test ; done > log

2.2) For HBase 0.94:
for test in `cat hbase-tests`; do ant clean test -Dhadoopversion=100 -Dtestcase=$test ; done > log

3) See the results:
cat log | grep "\[junit\]" 

Diff revision 2 (Latest)

1 2
1 2

  1. build.xml: Loading...
  2. ivy.xml: Loading...
build.xml
Revision 1a2128815d8cab33a24d9d97fe7392d53215f2f1 New Change
[20] 40 lines
[+20]
41
      <echo message="Use Hadoop 2.x by default" />
41
      <echo message="Use Hadoop 2.x by default" />
42
      <property name="hadoopversion" value="200" />
42
      <property name="hadoopversion" value="200" />
43
    </else>
43
    </else>
44
  </if>
44
  </if>
45

    
   
45

   
46
	<!-- Set variables per target hadoop version -->
46
  <!--

    
   
47
    Set default HBase profile if not set

    
   
48

   

    
   
49
    HBase 0.95 has broken one big monolithic archive into smaller pieces, so

    
   
50
    that the build dependencies are different between HBase 0.94 and 0.95.

    
   
51
    In addition, HBase since 0.95 is providing artifacts compatible with Hadoop 2.0.

    
   
52

   

    
   
53
    For HBase 0.95 on Hadoop 2.0.0:

    
   
54
    ant -Dhadoopversion=200 -Dhbaseprofile=95

    
   
55

   

    
   
56
    For HBase 0.95 on Hadoop 1.0.0:

    
   
57
    ant -Dhadoopversion=100 -Dhbaseprofile=95

    
   
58

   

    
   
59
    For HBase 0.94 on Hadoop 1.0.0:

    
   
60
    ant -Dhadoopversion=100

    
   
61

   

    
   
62
    For HBase 0.94 on Hadoop 2.0.0:

    
   
63
    - upstream HBase did not published Hadoop 2 compatible artifacts

    
   
64
  -->

    
   
65
  <if>

    
   
66
    <isset property="hbaseprofile" />

    
   
67
    <then>

    
   
68
      <echo message="Use HBase profile ${hbaseprofile}" />

    
   
69
    </then>

    
   
70
    <else>

    
   
71
      <echo message="Use HBase profile 0.94 by default" />

    
   
72
      <property name="hbaseprofile" value="94" />

    
   
73
    </else>

    
   
74
  </if>

    
   
75

   

    
   
76
	<!-- Set dependency versions per target Hadoop version -->
47
  <if>
77
  <if>
48
    <equals arg1="${hadoopversion}" arg2="20" />
78
    <equals arg1="${hadoopversion}" arg2="20" />
49
    <then>
79
    <then>
50
      <property name="hadoop.version" value="0.20.2-cdh3u1" />
80
      <property name="hadoop.version" value="0.20.2-cdh3u1" />
51
      <property name="hbase.version" value="0.90.3-cdh3u1" />
81
      <property name="hbase94.version" value="0.90.3-cdh3u1" />
52
      <property name="zookeeper.version" value="3.3.3-cdh3u1" />
82
      <property name="zookeeper.version" value="3.3.3-cdh3u1" />
53
      <property name="hadoop.version.full" value="0.20" />
83
      <property name="hadoop.version.full" value="0.20" />
54
      <property name="hcatalog.version" value="0.11.0" />
84
      <property name="hcatalog.version" value="0.11.0" />

    
   
85
        <property name="hbasecompatprofile" value="1" />
55
    </then>
86
    </then>
56

    
   
87

   
57
    <elseif>
88
    <elseif>
58
      <equals arg1="${hadoopversion}" arg2="23" />
89
      <equals arg1="${hadoopversion}" arg2="23" />
59
      <then>
90
      <then>
60
        <property name="hadoop.version" value="0.23.1" />
91
        <property name="hadoop.version" value="0.23.1" />
61
        <property name="hbase.version" value="0.92.0" />
92
        <property name="hbase94.version" value="0.92.0" />
62
        <property name="zookeeper.version" value="3.4.2" />
93
        <property name="zookeeper.version" value="3.4.2" />
63
        <property name="hadoop.version.full" value="0.23" />
94
        <property name="hadoop.version.full" value="0.23" />
64
        <property name="hcatalog.version" value="0.11.0" />
95
        <property name="hcatalog.version" value="0.11.0" />

    
   
96
        <property name="hbasecompatprofile" value="2" />
65
      </then>
97
      </then>
66
    </elseif>
98
    </elseif>
67

    
   
99

   
68
    <elseif>
100
    <elseif>
69
      <equals arg1="${hadoopversion}" arg2="100" />
101
      <equals arg1="${hadoopversion}" arg2="100" />
70
      <then>
102
      <then>
71
        <property name="hadoop.version" value="1.0.0" />
103
        <property name="hadoop.version" value="1.0.0" />
72
        <property name="hbase.version" value="0.92.0" />
104
        <property name="hbase94.version" value="0.92.0" />
73
        <property name="zookeeper.version" value="3.4.2" />
105
        <property name="zookeeper.version" value="3.4.2" />
74
        <property name="hadoop.version.full" value="1.0.0" />
106
        <property name="hadoop.version.full" value="1.0.0" />
75
        <property name="hcatalog.version" value="0.11.0" />
107
        <property name="hcatalog.version" value="0.11.0" />

    
   
108
        <property name="hbasecompatprofile" value="1" />
76
      </then>
109
      </then>
77
    </elseif>
110
    </elseif>
78

    
   
111

   
79
    <elseif>
112
    <elseif>
80
      <equals arg1="${hadoopversion}" arg2="200" />
113
      <equals arg1="${hadoopversion}" arg2="200" />
81
      <then>
114
      <then>
82
        <property name="hadoop.version" value="2.0.4-alpha" />
115
        <property name="hadoop.version" value="2.0.4-alpha" />
83
        <property name="hbase.version" value="0.94.2" />
116
        <property name="hbase94.version" value="0.94.2" />
84
        <property name="zookeeper.version" value="3.4.2" />
117
        <property name="zookeeper.version" value="3.4.2" />
85
        <property name="hadoop.version.full" value="2.0.4-alpha" />
118
        <property name="hadoop.version.full" value="2.0.4-alpha" />
86
        <property name="hcatalog.version" value="0.11.0" />
119
        <property name="hcatalog.version" value="0.11.0" />

    
   
120
        <property name="hbasecompatprofile" value="2" />
87
      </then>
121
      </then>
88
    </elseif>
122
    </elseif>
89

    
   
123

   
90
    <else>
124
    <else>
91
      <fail message="Unrecognized hadoopversion. Can only be 20, 23, 100 or 200." />
125
      <fail message="Unrecognized hadoopversion. Can only be 20, 23, 100 or 200." />
92
    </else>
126
    </else>
93
  </if>
127
  </if>
94

    
   
128

   

    
   
129
  <!-- Set dependency versions that are working with all Hadoop versions-->

    
   
130
  <property name="hbase95.version" value="0.95.2-hadoop${hbasecompatprofile}-SNAPSHOT" />

    
   
131

   
95
  <!-- Load system-wide and project-wide default properties set by
132
  <!-- Load system-wide and project-wide default properties set by
96
       the user, to avoid needing to override with -D. -->
133
       the user, to avoid needing to override with -D. -->
97
  <property file="${user.home}/build.properties" />
134
  <property file="${user.home}/build.properties" />
98
  <property file="${basedir}/build.properties" />
135
  <property file="${basedir}/build.properties" />
99

    
   
136

   
[+20] [20] 1245 lines
ivy.xml
Revision 63fdc80b6d8a0155ddc558126cf0082289f15372 New Change
 
  1. build.xml: Loading...
  2. ivy.xml: Loading...