Review Board 1.7.22


Request to review patch for SQOOP-954: Create Sqoop runtime scripts to run Sqoop on Windows

Review Request #10055 - Created March 20, 2013 and updated

Ahmed El Baz
trunk
SQOOP-954
Reviewers
Sqoop
sqoop-trunk
A patch implementing the Windows version of Sqoop run scripts. The scripts follow the same logic as there .sh counterparts.
One difference is to create a Jar which references all classpath elements in its Manifest, and provide that jar as the single jar needed for Sqoop. The reason here is that in some cases if the number of classpath elements is large, HADOOP_CLASSPATH gets very long which causes failures in Windows since there is a limit to command lines.
As a workaround, I added a step to wrap all jars in the classpath in a single jar, and then use that generated jar (this is also done in hadoop for Windows to handle similar issues)
I did this in a utility script "BuildJar" which can be used for other components as well.
This change is specific to Windows scripts, Linux scripts are not affected.

 

Diff revision 4 (Latest)

1 2 3 4
1 2 3 4

  1. bin/configure-sqoop.cmd: Loading...
  2. bin/sqoop.cmd: Loading...
  3. conf/sqoop-env-template.cmd: Loading...
bin/configure-sqoop.cmd
New File

    
   
1
:: Licensed to the Apache Software Foundation (ASF) under one or more

    
   
2
:: contributor license agreements.  See the NOTICE file distributed with

    
   
3
:: this work for additional information regarding copyright ownership.

    
   
4
:: The ASF licenses this file to You under the Apache License, Version 2.0

    
   
5
:: (the "License"); you may not use this file except in compliance with

    
   
6
:: the License.  You may obtain a copy of the License at

    
   
7
::

    
   
8
::

    
   
9
::     http://www.apache.org/licenses/LICENSE-2.0

    
   
10
::

    
   
11
:: Unless required by applicable law or agreed to in writing, software

    
   
12
:: distributed under the License is distributed on an "AS IS" BASIS,

    
   
13
:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

    
   
14
:: See the License for the specific language governing permissions and

    
   
15
:: limitations under the License.

    
   
16

   

    
   
17
if not exist %bin% (

    
   
18
  echo Error: Environment variable bin not defined.

    
   
19
  echo This is generally because this script should not be invoked directly. Use sqoop instead.

    
   
20
  exit /b 1

    
   
21
)

    
   
22

   

    
   
23
if not defined SQOOP_HOME (

    
   
24
  set SQOOP_HOME=%bin%\..

    
   
25
)

    
   
26

   

    
   
27
if not defined SQOOP_CONF_DIR (

    
   
28
  set SQOOP_CONF_DIR=%SQOOP_HOME%\conf

    
   
29
)

    
   
30

   

    
   
31
:: Call sqoop-env if it exists under SQOOP_CONF_DIR

    
   
32
if exist %SQOOP_CONF_DIR%\sqoop-env.cmd (

    
   
33
  call %SQOOP_CONF_DIR%\sqoop-env.cmd

    
   
34
)

    
   
35

   

    
   
36
:: Find paths to our dependency systems. If they are unset, use CDH defaults.

    
   
37

   

    
   
38
if not defined HADOOP_COMMON_HOME (

    
   
39
  if defined HADOOP_HOME (

    
   
40
    set HADOOP_COMMON_HOME=%HADOOP_HOME%

    
   
41
  ) else (

    
   
42
    :: Check: If we can't find our dependencies, give up here.

    
   
43
    echo Error: The environment variable HADOOP_HOME has not been defined.

    
   
44
    echo Please set HADOOP_HOME to the root of your Hadoop installation.

    
   
45
    exit /b 1

    
   
46
  )

    
   
47
)

    
   
48
if not defined HADOOP_MAPRED_HOME (

    
   
49
  if defined HADOOP_HOME (

    
   
50
    set HADOOP_MAPRED_HOME=%HADOOP_HOME%

    
   
51
  ) else (

    
   
52
    :: Check: If we can't find our dependencies, give up here.

    
   
53
    echo Error: The environment variable HADOOP_HOME has not been defined.

    
   
54
    echo Please set HADOOP_HOME to the root of your Hadoop installation.

    
   
55
    exit /b 1

    
   
56
  )

    
   
57
)

    
   
58
:: Check for HBase dependency

    
   
59
if not defined HBASE_HOME (

    
   
60
  if defined HBASE_VERSION (

    
   
61
    set HBASE_HOME=%HADOOP_HOME%\..\hbase-%HBASE_VERSION%

    
   
62
  ) else (

    
   
63
    echo Warning: HBASE_HOME and HBASE_VERSION not set.

    
   
64
  )

    
   
65
)

    
   
66

   

    
   
67
:: Check: If we can't find our dependencies, give up here.

    
   
68

   

    
   
69
:: Check: If HADOOP_COMMON_HOME path actually exists

    
   
70
if not exist %HADOOP_COMMON_HOME% (

    
   
71
  echo Error: HADOOP_COMMON_HOME does not exist!

    
   
72
  echo Please set HADOOP_COMMON_HOME to the root of your Hadoop installation.

    
   
73
  exit /b 1

    
   
74
)

    
   
75
:: Check: If HADOOP_MAPRED_HOME path actually exists

    
   
76
if not exist %HADOOP_MAPRED_HOME% (

    
   
77
  echo Error: HADOOP_MAPRED_HOME does not exist!

    
   
78
  echo Please set HADOOP_MAPRED_HOME to the root of your Hadoop installation.

    
   
79
  exit /b 1

    
   
80
)

    
   
81
if not exist "%HBASE_HOME%" (

    
   
82
  echo Warning: HBASE_HOME does not exist! HBase imports will fail.

    
   
83
  echo Please set HBASE_HOME to the root of your HBase installation.

    
   
84
)

    
   
85

   

    
   
86
:: Add sqoop dependencies to classpath

    
   
87
set SQOOP_CLASSPATH=

    
   
88

   

    
   
89
:: Where to find the main Sqoop jar

    
   
90
set SQOOP_JAR_DIR=%SQOOP_HOME%

    
   
91

   

    
   
92
:: If there's a "build" subdir, override with this, so we use

    
   
93
:: the newly-compiled copy.

    
   
94
if exist "%SQOOP_JAR_DIR%\build" (

    
   
95
  set SQOOP_JAR_DIR=%SQOOP_JAR_DIR%\build

    
   
96
)

    
   
97
call :add_dir_to_classpath %SQOOP_JAR_DIR%

    
   
98

   

    
   
99
if exist "%SQOOP_HOME%\lib" (

    
   
100
  call :add_dir_to_classpath %SQOOP_HOME%\lib

    
   
101
)

    
   
102

   

    
   
103
:: Add HBase to dependency list

    
   
104
if exist "%HBASE_HOME%" (

    
   
105
  call :add_dir_to_classpath %HBASE_HOME%

    
   
106
  call :add_dir_to_classpath %HBASE_HOME%\lib

    
   
107
)

    
   
108

   

    
   
109
if not defined ZOOCFGDIR (

    
   
110
  if defined ZOOKEEPER_CONF_DIR (

    
   
111
    set ZOOCFGDIR=%ZOOKEEPER_CONF_DIR%

    
   
112
  ) else (

    
   
113
  if defined ZOOKEEPER_HOME (

    
   
114
    set ZOOCFGDIR=%ZOOKEEPER_HOME%\conf

    
   
115
  ))

    
   
116
)

    
   
117

   

    
   
118
if "%ZOOCFGDIR%" NEQ "" (

    
   
119
  call :add_dir_to_classpath %ZOOCFGDIR%

    
   
120
)

    
   
121

   

    
   
122
call :add_dir_to_classpath %SQOOP_CONF_DIR%

    
   
123

   

    
   
124
:: If there's a build subdir, use Ivy-retrieved dependencies too.

    
   
125
if exist "%SQOOP_HOME%\build\ivy\lib\sqoop" (

    
   
126
  call :add_dir_to_classpath %SQOOP_HOME%\build\ivy\lib\sqoop

    
   
127
)

    
   
128

   

    
   
129
set HADOOP_CLASSPATH=%SQOOP_CLASSPATH%;%HADOOP_CLASSPATH%

    
   
130
if defined SQOOP_USER_CLASSPATH (

    
   
131
  :: User has elements to prepend to the classpath, forcibly overriding

    
   
132
  :: Sqoop's own lib directories.

    
   
133
  set HADOOP_CLASSPATH=%SQOOP_USER_CLASSPATH%;%HADOOP_CLASSPATH%

    
   
134
)

    
   
135

   

    
   
136
goto :eof

    
   
137

   

    
   
138
:: Function to add the given directory to the list of classpath directories

    
   
139
:: All jars under the given directory are added to the classpath

    
   
140
:add_dir_to_classpath

    
   
141
if not "%1"=="" (

    
   
142
  set SQOOP_CLASSPATH=!SQOOP_CLASSPATH!;%1\*

    
   
143
)

    
   
144
goto :eof
bin/sqoop.cmd
New File
 
conf/sqoop-env-template.cmd
New File
 
  1. bin/configure-sqoop.cmd: Loading...
  2. bin/sqoop.cmd: Loading...
  3. conf/sqoop-env-template.cmd: Loading...