Review Board 1.7.22


Flume startup script for windows

Review Request #13308 - Created Aug. 6, 2013 and updated

Roshan Naik
FLUME-1334
Reviewers
Flume
flume-git
Flume startup script for windows. 
The command line options have slightly changed (form unix script) to accomodate powershell requirements:

 - cannot have case sensitive options:  -d  (dry run) and -D  (java props). Similary -C & -c.
 - double hyphen is not allowed. So instead of --option, we use -option
 - multiple options with same name are not allowed. So instead of -classpath a=1 -classpath b=2 .. will change to -classpath "a=1;b=2"
 


Usage:

Usage: flume-ng <command> [options]...

commands:
  help                  display this help text
  agent                 run a Flume agent
  avro-client           run an avro Flume client
  version               show Flume version info

global options:
  -conf  <conf>                           use configs in <conf> directory
  -classpath,-C  ""value1;value2; ..""    append to the classpath
  -property ""name1=val;name2=val; ..""   sets a JDK system property value
  -dryrun,-d                              do not actually start Flume (test)

agent options:
  -conf-file,-f  <file>                specify a config file (required)
  -name,-n  <name>                     the name of this agent (required)

avro-client options:
  -host,-H  <host>           hostname to which events will be sent (required)
  -port,-p  <port>           port of the avro source (required)
  -filename  <file>          text file to stream to avro source [default: std input]
  -headerFile,-R <file>      headerFile containing headers as key/value pairs on each new line

 

Diff revision 2 (Latest)

1 2
1 2

  1. bin/flume-ng.ps1: Loading...
  2. bin/flume-ng.cmd: Loading...
bin/flume-ng.ps1
New File

    
   
1
#

    
   
2
# Licensed to the Apache Software Foundation (ASF) under one

    
   
3
# or more contributor license agreements.  See the NOTICE file

    
   
4
# distributed with this work for additional information

    
   
5
# regarding copyright ownership.  The ASF licenses this file

    
   
6
# to you under the Apache License, Version 2.0 (the

    
   
7
# "License"); you may not use this file except in compliance

    
   
8
# with the License.  You may obtain a copy of the License at

    
   
9
#

    
   
10
#   http://www.apache.org/licenses/LICENSE-2.0

    
   
11
#

    
   
12
# Unless required by applicable law or agreed to in writing,

    
   
13
# software distributed under the License is distributed on an

    
   
14
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

    
   
15
# KIND, either express or implied.  See the License for the

    
   
16
# specific language governing permissions and limitations

    
   
17
# under the License.

    
   
18
#

    
   
19

   

    
   
20
param(

    
   
21
    [Parameter(Position=0, Mandatory=$true)]

    
   
22
    [ValidateSet('help','agent','avro-client','version')] $command,

    
   
23
    [Parameter()] [string] $conf,

    
   
24
    [Parameter()] [Alias('C')] [string] $classPath,

    
   
25
    [Parameter()] [Alias('property')] [string] $javaProperties,

    
   
26
    [Parameter()] [Alias('d')] [switch] $dryrun,

    
   
27
    [Parameter()] [Alias('plugins-path')] [string] $pluginsPath,

    
   
28

   

    
   
29
    [Parameter()] [Alias('f',"conf-file")] [string] $confFile,

    
   
30
    [Parameter()] [Alias('n')] [string] $name,

    
   
31

   

    
   
32
    [Parameter()] [string] $rpcProps,

    
   
33
    [Parameter()] [Alias('H',"host")] [string] $avroHost,

    
   
34
    [Parameter()] [Alias('p',"port")] [string] $avroPort,

    
   
35
    [Parameter()] [Alias("dirname")]  [string] $avroDirName,

    
   
36
    [Parameter()] [Alias("filename")] [string] $avroFileName,

    
   
37
    [Parameter()] [Alias('R',"headerFile")] [string] $avroHeaderFile

    
   
38
)

    
   
39

   

    
   
40
# when invoked from cmd shell, array arguments are treated as array with one string value

    
   
41
# so we accept classpath and javaProperty as string argument and split explicitly into array

    
   
42

   

    
   
43
if ($classPath -ne "") {

    
   
44
   $classPathArr = $classPath.Split(";")

    
   
45
} else {

    
   
46
   $classPathArr = @()

    
   
47
}

    
   
48

   

    
   
49
if($javaProperties -ne "") {

    
   
50
   $javaPropertiesArr = $javaProperties.Split(";")  # properties may have embedded comma, so using ; as delim

    
   
51
} else {

    
   
52
   $javaPropertiesArr = @()

    
   
53
}

    
   
54

   

    
   
55
# FUNCTIONS

    
   
56
Function PrintUsage {

    
   
57
    Write-Host "

    
   
58
Usage: flume-ng <command> [options]...

    
   
59

   

    
   
60
commands:

    
   
61
  help                  display this help text

    
   
62
  agent                 run a Flume agent

    
   
63
  avro-client           run an avro Flume client

    
   
64
  version               show Flume version info

    
   
65

   

    
   
66
global options:

    
   
67
  -conf  <conf>                           use configs in <conf> directory

    
   
68
  -classpath,-C  ""value1;value2; ..""    append to the classpath

    
   
69
  -property ""name1=val;name2=val; ..""   sets a JDK system property value

    
   
70
  -dryrun,-d                              do not actually start Flume (test)

    
   
71
  -plugins-path ""dir1;dir2""             semi-colon-separated list of plugins.d directories. See the

    
   
72
                                          plugins.d section in the user guide for more details.

    
   
73
                                          Default: \$FLUME_HOME/plugins.d

    
   
74

   

    
   
75
agent options:

    
   
76
  -conf-file,-f  <file>                specify a config file (required)

    
   
77
  -name,-n  <name>                     the name of this agent (required)

    
   
78

   

    
   
79
avro-client options:

    
   
80
  -rpcProps <file>           RPC client properties file with server connection params

    
   
81
  -host,-H  <host>           hostname to which events will be sent (required)

    
   
82
  -port,-p  <port>           port of the avro source (required)

    
   
83
  -dirname <dir>             directory to stream to avro source

    
   
84
  -filename  <file>          text file to stream to avro source [default: std input]

    
   
85
  -headerFile,-R <file>      headerFile containing headers as key/value pairs on each new line

    
   
86

   

    
   
87
  Either --rpcProps or both --host and --port must be specified.

    
   
88

   

    
   
89
Note that if <conf> directory is specified, then it is always included first

    
   
90
in the classpath."

    
   
91
}

    
   
92

   

    
   
93
Function EnumerateJars($path) {

    
   
94
   return Get-ChildItem $path -Filter *.jar | ? { "$_" -notmatch "slf4j-(api|log4j12).*jar" } | % { $_.FullName}

    
   
95
}

    
   
96

   

    
   
97
Function GetHadoopHome {

    
   
98
    if($env:HADOOP_PREFIX) {

    
   
99
        $hadoopBin = "$env:HADOOP_PREFIX\bin;"

    
   
100
    } elseif ($env:HADOOP_HOME) {

    
   
101
        $hadoopBin = "$env:HADOOP_HOME\bin;"

    
   
102
    }

    
   
103

   

    
   
104
    #Searches for hadoop.cmd in the HADOOP_HOME, current directory and path

    
   
105
    [String[]] $hadoopPaths = ("$hadoopBin;.;$env:PATH").Split(";") |

    
   
106
                                 ? { "$_" -ne "" -and (Test-Path $_ )} |

    
   
107
                                 ? { Test-Path ( Join-Path $_ "hadoop.cmd" ) }

    
   
108
    if($hadoopPaths -ne $Null ) {

    
   
109
       $binDir = $hadoopPaths[0] + "\.."

    
   
110
       return  Resolve-Path $binDir

    
   
111
    }

    
   
112
    Write-Host "WARN: HADOOP_PREFIX or HADOOP_HOME not found"

    
   
113
    return $Null

    
   
114
}

    
   
115

   

    
   
116
Function GetHbaseHome() {

    
   
117
    if($env:HBASE_HOME) {

    
   
118
        return $env:HBASE_HOME

    
   
119
    }

    
   
120

   

    
   
121
    #Searches for hbase.cmd in the HBASE_HOME, current directory and path

    
   
122
    [String[]] $hbasePaths = (".;$env:PATH").Split(";") |

    
   
123
                               ? { "$_" -ne "" -and (Test-Path $_) } |

    
   
124
                               ? { Test-Path (Join-Path $_ "hbase.cmd") }

    
   
125

   

    
   
126
    if($hbasePaths -ne $Null) {

    
   
127
        return $hbasePaths[0]

    
   
128
    }

    
   
129
    Write-Host "WARN: HBASE_HOME not found"

    
   
130
    return $Null

    
   
131
}

    
   
132

   

    
   
133

   

    
   
134
Function GetJavaLibraryPath ($cmd, $flumeLibDir) {

    
   
135
    $flumeCoreJar = EnumerateJars( $flumeLibDir ) | ? { $_ -match "flume-ng-core.*jar" }

    
   
136
    $output = & "$cmd" -classpath $flumeCoreJar org.apache.flume.tools.GetJavaProperty java.library.path

    
   
137

   

    
   
138
    # look for the line that has the desired property value

    
   
139
    if( $output -match "(?m)^java\.library\.path=(.*)$" ) {

    
   
140
      return $Matches[1].split(";") | % { $_ -replace "\\$" , ""}  # trim trailing \ char

    
   
141
    }

    
   
142
    return @();

    
   
143
}

    
   
144

   

    
   
145

   

    
   
146
Function GetClassPath ($cmd) {

    
   
147
    $output = & "$cmd" "classpath"

    
   
148
	return $output;

    
   
149
}

    
   
150

   

    
   
151
Function GetJavaPath {

    
   
152
    if ($env:JAVA_HOME) {

    
   
153
        return "$env:JAVA_HOME\bin\java.exe" }ss

    
   
154
    Write-Host "WARN: JAVA_HOME not set"

    
   
155
    return '"' + (Resolve-Path "java.exe").Path + '"'

    
   
156
}

    
   
157

   

    
   
158

   

    
   
159
function runFlume($javaClassPath, $javaLibraryPath, $javaOptions, $class, $javaProcessArgumentList)  {

    
   
160
  [string]$javaPath = GetJavaPath

    
   
161
  [string]$fullJavaCommand = "-classpath $javaClassPath -Djava.library.path=$javaLibraryPath $javaOptions $class $javaProcessArgumentList"

    
   
162
  if ($dryrun) {

    
   
163
    Write-Host 'Dry run mode enabled (will not actually initiate startup)'

    
   
164
    Write-Host "$javaPath $fullJavaCommand"

    
   
165
  } else {

    
   
166
    Write-Host "

    
   
167
  Running FLUME $command :

    
   
168
    class: $class

    
   
169
    arguments: $javaProcessArgumentList

    
   
170
    "

    
   
171

   

    
   
172
    $ErrorActionPreference = "Continue"

    
   
173
    $x = Start-Process $javaPath -ArgumentList "$fullJavaCommand" -Wait -NoNewWindow

    
   
174
  }

    
   
175
}

    
   
176

   

    
   
177

   

    
   
178
# The script will terminate if any steps fail

    
   
179
$ErrorActionPreference = "Stop"

    
   
180

   

    
   
181
#SWITCH FOR THE DIFFERENT COMMANDS

    
   
182
[string] $javaProcessArgumentList

    
   
183
switch ($command)

    
   
184
{

    
   
185
  'help' {

    
   
186
    PrintUsage

    
   
187
    return }

    
   
188

   

    
   
189
  'agent' {

    
   
190
    if (!$Name) {

    
   
191
      PrintUsage

    
   
192
      Write-Host "ERROR: Name parameter missing"

    
   
193
      return

    
   
194
    }

    
   
195
    if (!$ConfFile) {

    
   
196
      PrintUsage

    
   
197
      Write-Host "ERROR: ConfFile parameter missing"

    
   
198
      return

    
   
199
    }

    
   
200
    $class='org.apache.flume.node.Application'

    
   
201
    $confFile = '"' + (Resolve-Path $confFile).Path + '"'

    
   
202
    $javaProcessArgumentList = "-n $name -f $confFile"  }

    
   
203

   

    
   
204
  'avro-client' {

    
   
205
    $class='org.apache.flume.client.avro.AvroCLIClient'

    
   
206
    if("$rpcProps" -eq "") {

    
   
207
      if (!$AvroHost) {

    
   
208
        PrintUsage

    
   
209
        Write-Host "ERROR: Avro Host parameter missing"

    
   
210
        return

    
   
211
      }

    
   
212
      if (!$AvroPort) {

    
   
213
        PrintUsage

    
   
214
        Write-Host "ERROR: Avro Port parameter missing"

    
   
215
        return

    
   
216
      }

    
   
217
      $javaProcessArgumentList = " -H $AvroHost -p $AvroPort"

    
   
218
    } else {

    
   
219
      $javaProcessArgumentList += " --rpcProps ""$rpcProps"""

    
   
220
    }

    
   
221

   

    
   
222
    if ($avroHeaderFile) {

    
   
223
      $avroHeaderFile = (Resolve-Path $avroHeaderFile).Path

    
   
224
      $javaProcessArgumentList += " -R $avroHeaderFile" }

    
   
225
    if ($avroFileName) {

    
   
226
      $avroFileName = (Resolve-Path $avroFileName).Path

    
   
227
      $javaProcessArgumentList += " -F $avroFileName" }

    
   
228
    if($avroDirName) {

    
   
229
      $avroDirName = (Resolve-Path $avroDirName).Path

    
   
230
      $javaProcessArgumentList += " --dirname ""$avroDirName""" } }

    
   
231

   

    
   
232
  'version' {

    
   
233
    $class='org.apache.flume.tools.VersionInfo'

    
   
234
    $javaProcessArgumentList = "" }

    
   
235

   

    
   
236
  default {

    
   
237
    PrintUsage

    
   
238
    Write-Host "ERROR: Invalid command '$command'"

    
   
239
    return }

    
   
240
}

    
   
241

   

    
   
242
$FlumeHome = $null

    
   
243
if($env:FLUME_HOME) {

    
   
244
    $FlumeHome = $env:FLUME_HOME

    
   
245
}  else {

    
   
246
    $ScriptPath =  Split-Path -Parent $MyInvocation.MyCommand.Path

    
   
247
    $FlumeHome =  Split-Path -Parent $ScriptPath

    
   
248
}

    
   
249

   

    
   
250
########### Source flume-env.ps1 ##############

    
   
251

   

    
   
252
# allow users to override the default env vars via conf\flume-env.ps1

    
   
253
if( "$conf" -eq "" ) {

    
   
254
    if( Test-path ("$FlumeHome\conf") ) {

    
   
255
      $conf = "$FlumeHome\conf"

    
   
256
      Write-Host "WARN: Config directory not set. Defaulting to $conf"

    
   
257
    }

    
   
258
}

    
   
259
if ( "$conf" -ne "" )  {

    
   
260
  Write-Host "Sourcing environment configuration script $conf\flume-env.ps1"

    
   
261
  if ( Test-path  "$conf\flume-env.ps1" ) {

    
   
262
     . "$conf\flume-env.ps1"

    
   
263
  }  else {

    
   
264
     Write-Host "WARN: Did not find $conf\flume-env.ps1"

    
   
265
  }

    
   
266
}  else {

    
   
267
  Write-Host "WARN: No configuration directory found! Use --conf <dir> to set."

    
   
268
}

    
   
269

   

    
   
270
########### Setup JAVA_OPTS ##############

    
   
271

   

    
   
272
[string]$javaOptions="$JAVA_OPTS"

    
   
273
foreach ($opt in $javaPropertiesArr) {

    
   
274
  $javaOptions = "$javaOptions -D$opt"

    
   
275
}

    
   
276

   

    
   
277
########### Setup Classpath ###############

    
   
278
# flume\conf ; flume_home\lib\* ; cmdline ; env.ps1 ; plugins.d ; hadoop.cpath ; hbase.cpath

    
   
279

   

    
   
280
[string]$javaClassPath='"' + $conf + '"'

    
   
281
[string]$flumeLibJars=""

    
   
282
[string]$flumeLibDir = Resolve-Path "$FlumeHome\lib"

    
   
283

   

    
   
284
# Add FlumeHome\lib\* to class path

    
   
285
$javaClassPath = "$javaClassPath;""$flumeLibDir\*"""

    
   
286
$flumeLibJars = "$flumeLibDir\*"""

    
   
287

   

    
   
288
# Add classpath from cmd line & FLUME_CLASSPATH in flume-env.ps1

    
   
289
if ( $FLUME_CLASSPATH )  {

    
   
290
  $classPathArr = $FLUME_CLASSPATH.Split(";")

    
   
291
}

    
   
292
foreach ($path in $classPathArr) {

    
   
293
  $fullPath = (Resolve-Path $path).Path

    
   
294
  $javaClassPath = "$javaClassPath;""$fullPath"""

    
   
295
}

    
   
296

   

    
   
297
$javaLibraryPath = ""

    
   
298

   

    
   
299
# Add plugins.d into classpath and libpath

    
   
300
if ("$pluginsPath" -eq "") {

    
   
301
   $pluginsPath = "$FlumeHome\plugins.d"

    
   
302
}

    
   
303

   

    
   
304
foreach($plugin in  $pluginsPath.Split(";") )  {

    
   
305
  if ( Test-path "$plugin" ) {

    
   
306
    $pluginTmp =

    
   
307
        ( (Get-ChildItem "$plugin\*\lib")  + (Get-ChildItem "$plugin\*\libext") ) -join "\*"";"""

    
   
308
    if( "$pluginTmp" -ne "" ) {

    
   
309
      $javaClassPath="$javaClassPath;""" + $pluginTmp + "\*"";"

    
   
310
    }

    
   
311

   

    
   
312
    $javaLibraryPathTmp = (@(Get-ChildItem "$plugin\*\native") -join "\*"";""")

    
   
313
    if( "$javaLibraryPathTmp" -ne "" )  {

    
   
314
       $javaLibraryPath=  "$javaLibraryPath""" + "$javaLibraryPathTmp" + "\*"";"

    
   
315
    }

    
   
316
  }

    
   
317
}

    
   
318

   

    
   
319
# Add Hadoop classpath &  java.library.path

    
   
320
$hadoopHome = GetHadoopHome

    
   
321
if("$hadoopHome" -ne "") {

    
   
322
  $hadoopCmd = "$hadoopHome\bin\hadoop.cmd"

    
   
323
  if( Test-Path $hadoopCmd ) {

    
   
324
    Write-Host "Including Hadoop libraries found in ($hadoopHome) for DFS access" 

    
   
325
    $javaClassPath = "$javaClassPath;""$hadoopHome\conf""";

    
   
326
    foreach ($path in  GetClassPath $hadoopCmd) {

    
   
327
      $javaClassPath = "$javaClassPath;""$path"""

    
   
328
    }

    
   
329

   

    
   
330
    foreach ( $path in GetJavaLibraryPath $hadoopCmd  $flumeLibDir ) {

    
   
331
       $javaLibraryPath = "$javaLibraryPath""$path"";"

    
   
332
    }

    
   
333
  } else {

    
   
334
    Write-Host "WARN: $hadoopCmd not be found. Unable to include Hadoop's classpath & java.library.path"

    
   
335
  }

    
   
336
} else {

    
   
337
  Write-Host "WARN: HADOOP_PREFIX not set. Unable to include Hadoop's classpath & java.library.path"

    
   
338
}

    
   
339

   

    
   
340
# Add HBase classpath  & java.library.path

    
   
341
$hbaseHome = GetHbaseHome

    
   
342
if( "$hbaseHome" -ne "" ) {

    
   
343
    $hbaseCmd = "$hbaseHome\bin\hbase.cmd"

    
   
344
    if( Test-Path $hbaseCmd ) {

    
   
345
      Write-Host "Including HBase libraries found via ($hbaseHome) for HBase access"

    
   
346
      foreach ( $path in GetClassPath $hbaseCmd ) {

    
   
347
          $javaClassPath = "$javaClassPath;""$path"""

    
   
348
      }

    
   
349
      $javaClassPath = "$javaClassPath;""$hbaseHome\conf"""

    
   
350

   

    
   
351
      foreach ( $path in GetJavaLibraryPath $hbaseCmd  $flumeLibDir ) {

    
   
352
          $javaLibraryPath = "$javaLibraryPath""$path"";"

    
   
353
      }

    
   
354
    } else {

    
   
355
        Write-Host "WARN: $hbaseCmd not be found. Unable to include HBase classpath and java.library.path"

    
   
356
    }

    
   
357
}

    
   
358

   

    
   
359
runFlume $javaClassPath $javaLibraryPath $javaOptions $class $javaProcessArgumentList
bin/flume-ng.cmd
New File
 
  1. bin/flume-ng.ps1: Loading...
  2. bin/flume-ng.cmd: Loading...