Review Board 1.7.22


FLUME-1125: flume-ng script allows flume-env.sh to clobber some command-line arguments

Review Request #4759 - Created April 17, 2012 and submitted

Mike Percy
FLUME-1125
Reviewers
Flume
flume-git
We shouldn't allow the flume-env.sh variables to override command-line parameters. This patch fixes that issue.
This also adds support for a --dryrun parameter in order to test the variations and cleans up the flume-env.sh.template file a little bit.
The following vars are longer "clobberable" by flume-env.sh: FLUME_CLASSPATH, JAVA_OPTS
Manually tested various combinations.

Diff revision 1 (Latest)

  1. bin/flume-ng: Loading...
  2. conf/flume-env.sh.template: Loading...
bin/flume-ng
Revision 145db32 New Change
[20] 102 lines
[+20]
103
  avro-client           run an avro Flume client
103
  avro-client           run an avro Flume client
104

    
   
104

   
105
global options:
105
global options:
106
  --conf,-c <conf>      use configs in <conf> directory
106
  --conf,-c <conf>      use configs in <conf> directory
107
  --classpath,-C <cp>   append to the classpath
107
  --classpath,-C <cp>   append to the classpath

    
   
108
  --dryrun,-d           do not actually start Flume, just print the command
108
  -Dproperty=value      sets a JDK system property value
109
  -Dproperty=value      sets a JDK system property value
109

    
   
110

   
110
agent options:
111
agent options:
111
  --conf-file,-f        specify a config file (required)
112
  --conf-file,-f        specify a config file (required)
112
  --name,-n             the name of this agent (required)
113
  --name,-n             the name of this agent (required)
[+20] [20] 20 lines
[+20]
133
  else
134
  else
134
    error "Must specify flume application class" 1
135
    error "Must specify flume application class" 1
135
  fi
136
  fi
136

    
   
137

   
137
  set -x
138
  set -x
138
  exec $JAVA_HOME/bin/java $JAVA_OPTS -cp "$FLUME_CLASSPATH" \
139
  $EXEC $JAVA_HOME/bin/java $JAVA_OPTS -cp "$FLUME_CLASSPATH" \
139
      -Djava.library.path=$FLUME_JAVA_LIBRARY_PATH "$FLUME_APPLICATION_CLASS" $*
140
      -Djava.library.path=$FLUME_JAVA_LIBRARY_PATH "$FLUME_APPLICATION_CLASS" $*
140
}
141
}
141

    
   
142

   
142
################################
143
################################
143
# main
144
# main
144
################################
145
################################
145

    
   
146

   
146
# set default params
147
# set default params
147
FLUME_CLASSPATH=""
148
FLUME_CLASSPATH=""
148
FLUME_JAVA_LIBRARY_PATH=""
149
FLUME_JAVA_LIBRARY_PATH=""
149
JAVA_OPTS="-Xmx20m"
150
JAVA_OPTS="-Xmx20m"
150

    
   
151

   
151
opt_conf=""
152
opt_conf=""

    
   
153
opt_classpath=""

    
   
154
opt_java_props=""

    
   
155
opt_dryrun=""
152

    
   
156

   
153
mode=$1
157
mode=$1
154
shift
158
shift
155

    
   
159

   
156
case "$mode" in
160
case "$mode" in
[+20] [20] 29 lines
[+20]
186
      opt_conf=$1
190
      opt_conf=$1
187
      shift
191
      shift
188
      ;;
192
      ;;
189
    --classpath|-C)
193
    --classpath|-C)
190
      [ -n "$1" ] || error "Option --classpath requires an argument" 1
194
      [ -n "$1" ] || error "Option --classpath requires an argument" 1
191
      FLUME_CLASSPATH=$1
195
      opt_classpath=$1
192
      shift
196
      shift
193
      ;;
197
      ;;

    
   
198
    --dryrun|-d)

    
   
199
      opt_dryrun="1"

    
   
200
      ;;
194
    -D*)
201
    -D*)
195
      JAVA_OPTS="${JAVA_OPTS} $arg"
202
      opt_java_props="$opt_java_props $arg"
196
      ;;
203
      ;;
197
    *)
204
    *)
198
      args="$args $arg"
205
      args="$args $arg"
199
      ;;
206
      ;;
200
  esac
207
  esac
[+20] [20] 6 lines
[+20]
207

    
   
214

   
208
# allow users to override the default env vars via conf/flume-env.sh
215
# allow users to override the default env vars via conf/flume-env.sh
209
if [ -z "$opt_conf" ]; then
216
if [ -z "$opt_conf" ]; then
210
  warn "No configuration directory set! Use --conf <dir> to override."
217
  warn "No configuration directory set! Use --conf <dir> to override."
211
elif [ -f "$opt_conf/flume-env.sh" ]; then
218
elif [ -f "$opt_conf/flume-env.sh" ]; then

    
   
219
  info "Sourcing environment configuration script $opt_conf/flume-env.sh"
212
  source "$opt_conf/flume-env.sh"
220
  source "$opt_conf/flume-env.sh"
213
fi
221
fi
214

    
   
222

   

    
   
223
# append command-line java options to stock or env script JAVA_OPTS

    
   
224
if [ -n "${opt_java_props}" ]; then

    
   
225
  JAVA_OPTS="${JAVA_OPTS} ${opt_java_props}"

    
   
226
fi

    
   
227

   

    
   
228
# prepend command-line classpath to env script classpath

    
   
229
if [ -n "${opt_classpath}" ]; then

    
   
230
  if [ -n "${FLUME_CLASSPATH}" ]; then

    
   
231
    FLUME_CLASSPATH="${opt_classpath}:${FLUME_CLASSPATH}"

    
   
232
  else

    
   
233
    FLUME_CLASSPATH="${opt_classpath}"

    
   
234
  fi

    
   
235
fi

    
   
236

   

    
   
237
# prepend flume jars to the specified classpath (if any)
Moved from 238

    
   
238
if [ -n "${FLUME_CLASSPATH}" ] ; then
Moved from 239

    
   
239
  FLUME_CLASSPATH="${FLUME_HOME}/lib/*:$FLUME_CLASSPATH"
Moved from 240

    
   
240
else
Moved from 241

    
   
241
  FLUME_CLASSPATH="${FLUME_HOME}/lib/*"
Moved from 242

    
   
242
fi

    
   
243

   
215
# find java
244
# find java
216
if [ -z "${JAVA_HOME}" ] ; then
245
if [ -z "${JAVA_HOME}" ] ; then
217
  echo "Warning: JAVA_HOME not set!"
246
  echo "Warning: JAVA_HOME not set!"
218
  # Try to use Bigtop to autodetect JAVA_HOME if it's available
247
  # Try to use Bigtop to autodetect JAVA_HOME if it's available
219
  if [ -e /usr/libexec/bigtop-detect-javahome ]; then
248
  if [ -e /usr/libexec/bigtop-detect-javahome ]; then
[+20] [20] 12 lines
[+20]
232
# figure out where the flume distribution is
261
# figure out where the flume distribution is
233
if [ -z "${FLUME_HOME}" ] ; then
262
if [ -z "${FLUME_HOME}" ] ; then
234
  FLUME_HOME=$(cd $(dirname $0)/..; pwd)
263
  FLUME_HOME=$(cd $(dirname $0)/..; pwd)
235
fi
264
fi
236

    
   
265

   
237
# Append to the classpath

   
238
if [ -n "${FLUME_CLASSPATH}" ] ; then
Moved to 238

   
239
  FLUME_CLASSPATH="${FLUME_HOME}/lib/*:$FLUME_CLASSPATH"
Moved to 239

   
240
else
Moved to 240

   
241
  FLUME_CLASSPATH="${FLUME_HOME}/lib/*"
Moved to 241

   
242
fi
Moved to 242

   
243

    
   

   
244
# look for hadoop libs
266
# look for hadoop libs
245
add_hadoop_paths
267
add_hadoop_paths
246

    
   
268

   
247
# prepend conf dir to classpath
269
# prepend conf dir to classpath
248
if [ -n "$opt_conf" ]; then
270
if [ -n "$opt_conf" ]; then
249
  FLUME_CLASSPATH="$opt_conf:$FLUME_CLASSPATH"
271
  FLUME_CLASSPATH="$opt_conf:$FLUME_CLASSPATH"
250
fi
272
fi
251

    
   
273

   

    
   
274
# allow dryrun

    
   
275
EXEC="exec"

    
   
276
if [ -n "${opt_dryrun}" ]; then

    
   
277
  warn "Dryrun mode enabled (will not actually initiate startup)"

    
   
278
  EXEC="echo"

    
   
279
fi

    
   
280

   
252
# finally, invoke the appropriate command
281
# finally, invoke the appropriate command
253
if [ -n "$opt_agent" ] ; then
282
if [ -n "$opt_agent" ] ; then
254
  run_flume $FLUME_AGENT_CLASS $args
283
  run_flume $FLUME_AGENT_CLASS $args
255
elif [ -n "$opt_avro_client" ] ; then
284
elif [ -n "$opt_avro_client" ] ; then
256
  run_flume $FLUME_AVRO_CLIENT_CLASS $args
285
  run_flume $FLUME_AVRO_CLIENT_CLASS $args
257
else
286
else
258
  error "This message should never appear" 1
287
  error "This message should never appear" 1
259
fi
288
fi
260

    
   
289

   
261
exit 0
290
exit 0
conf/flume-env.sh.template
Revision 0388ef9 New Change
 
  1. bin/flume-ng: Loading...
  2. conf/flume-env.sh.template: Loading...