Review Board 1.7.22


FLUME-1285 FileChannel has a dependency on Hadoop IO classes

Review Request #10829 - Created April 29, 2013 and updated

Israel Ekpo
flume-1.4
FLUME-1285
Reviewers
Flume
flume-git
Modified and Applied Fix from [~chris.nagy] to 1.4 branch

Tested on Ubuntu 12.04 with 1.6.0_33 64-bit

modified:   flume-ng-channels/flume-file-channel/pom.xml
modified:   flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FlumeEvent.java
modified:   flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java
new file:   flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Writable.java
new file:   flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/WritableUtils.java
modified:   flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
Tested on Ubuntu 12.04 with Java 1.6.0_33 64-bit

- Unit Tests for Flume NG file-based channel were all successful with these changes.
- Created and deployed local build successfully with the following settings

agent.sources = seqGenSrc
agent.channels = fileChannel
agent.sinks = loggerSink


agent.sources.seqGenSrc.type = seq
agent.sources.seqGenSrc.channels = fileChannel

agent.channels.fileChannel.type = file
agent.channels.c1.checkpointDir	= /flume-staging/file-channel/checkpoint
agent.channels.c1.dataDirs = /flume-staging/file-channel/data


agent.sinks.loggerSink.type = logger
agent.sinks.loggerSink.channel = fileChannel

Agent was started as follows

$ bin/flume-ng agent --conf-file conf/flume.properties --conf conf --name agent

Agent started up and loaded File Channel successfully.

Agent was also able to shut down successfully.
flume-ng-channels/flume-file-channel/pom.xml
Revision 2408447 New Change

    
   
1
<?xml version="1.0"?>
1
<!--
2
<!--
2
 Licensed to the Apache Software Foundation (ASF) under one
3
 Licensed to the Apache Software Foundation (ASF) under one
3
 or more contributor license agreements.  See the NOTICE file
4
 or more contributor license agreements.  See the NOTICE file
4
 distributed with this work for additional information
5
 distributed with this work for additional information
5
 regarding copyright ownership.  The ASF licenses this file
6
 regarding copyright ownership.  The ASF licenses this file
[+20] [20] 8 lines
[+20]
14
 "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
 "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
 KIND, either express or implied.  See the License for the
16
 KIND, either express or implied.  See the License for the
16
 specific language governing permissions and limitations
17
 specific language governing permissions and limitations
17
 under the License.
18
 under the License.
18
-->
19
-->
19

    
   
20
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

   
21
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

   
22

    
   

   
23
  <modelVersion>4.0.0</modelVersion>
21
  <modelVersion>4.0.0</modelVersion>
24
  <parent>
22
  <parent>
25
    <artifactId>flume-ng-channels</artifactId>
23
    <artifactId>flume-ng-channels</artifactId>
26
    <groupId>org.apache.flume</groupId>
24
    <groupId>org.apache.flume</groupId>
27
    <version>1.4.0-SNAPSHOT</version>
25
    <version>1.4.0-SNAPSHOT</version>
28
  </parent>
26
  </parent>
29

    
   

   
30
  <groupId>org.apache.flume.flume-ng-channels</groupId>
27
  <groupId>org.apache.flume.flume-ng-channels</groupId>
31
  <artifactId>flume-file-channel</artifactId>
28
  <artifactId>flume-file-channel</artifactId>
32
  <name>Flume NG file-based channel</name>
29
  <name>Flume NG file-based channel</name>
33

    
   

   
34
  <build>
30
  <build>
35
    <plugins>
31
    <plugins>
36
      <plugin>
32
      <plugin>
37
        <groupId>org.apache.rat</groupId>
33
        <groupId>org.apache.rat</groupId>
38
        <artifactId>apache-rat-plugin</artifactId>
34
        <artifactId>apache-rat-plugin</artifactId>
39
      </plugin>
35
      </plugin>
40
    </plugins>
36
    </plugins>
41
  </build>
37
  </build>
42

    
   

   
43
  <dependencies>
38
  <dependencies>
44

    
   

   
45
    <dependency>
39
    <dependency>
46
      <groupId>org.apache.flume</groupId>
40
      <groupId>org.apache.flume</groupId>
47
      <artifactId>flume-ng-sdk</artifactId>
41
      <artifactId>flume-ng-sdk</artifactId>
48
    </dependency>
42
    </dependency>
49

    
   

   
50
    <dependency>
43
    <dependency>
51
      <groupId>org.apache.flume</groupId>
44
      <groupId>org.apache.flume</groupId>
52
      <artifactId>flume-ng-configuration</artifactId>
45
      <artifactId>flume-ng-configuration</artifactId>
53
    </dependency>
46
    </dependency>
54

    
   

   
55
    <dependency>
47
    <dependency>
56
      <groupId>org.apache.flume</groupId>
48
      <groupId>org.apache.flume</groupId>
57
      <artifactId>flume-ng-core</artifactId>
49
      <artifactId>flume-ng-core</artifactId>
58
    </dependency>
50
    </dependency>
59

    
   

   
60
    <dependency>
51
    <dependency>
61
      <groupId>org.slf4j</groupId>
52
      <groupId>org.slf4j</groupId>
62
      <artifactId>slf4j-api</artifactId>
53
      <artifactId>slf4j-api</artifactId>
63
    </dependency>
54
    </dependency>
64

    
   

   
65
    <dependency>
55
    <dependency>
66
      <groupId>commons-collections</groupId>
56
      <groupId>commons-collections</groupId>
67
      <artifactId>commons-collections</artifactId>
57
      <artifactId>commons-collections</artifactId>
68
    </dependency>
58
    </dependency>
69

    
   

   
70
    <dependency>
59
    <dependency>
71
      <groupId>commons-lang</groupId>
60
      <groupId>commons-lang</groupId>
72
      <artifactId>commons-lang</artifactId>
61
      <artifactId>commons-lang</artifactId>
73
    </dependency>
62
    </dependency>
74

    
   

   
75
    <dependency>
63
    <dependency>
76
      <groupId>junit</groupId>
64
      <groupId>junit</groupId>
77
      <artifactId>junit</artifactId>
65
      <artifactId>junit</artifactId>
78
      <scope>test</scope>
66
      <scope>test</scope>
79
    </dependency>
67
    </dependency>
80

    
   

   
81
    <dependency>
68
    <dependency>
82
      <groupId>org.mockito</groupId>
69
      <groupId>org.mockito</groupId>
83
      <artifactId>mockito-all</artifactId>
70
      <artifactId>mockito-all</artifactId>
84
      <scope>test</scope>
71
      <scope>test</scope>
85
    </dependency>
72
    </dependency>
86

    
   

   
87
    <dependency>
73
    <dependency>
88
      <groupId>org.easytesting</groupId>
74
      <groupId>org.easytesting</groupId>
89
      <artifactId>fest-reflect</artifactId>
75
      <artifactId>fest-reflect</artifactId>
90
      <scope>test</scope>
76
      <scope>test</scope>
91
    </dependency>
77
    </dependency>
92

    
   

   
93
    <dependency>
78
    <dependency>
94
      <groupId>org.slf4j</groupId>
79
      <groupId>org.slf4j</groupId>
95
      <artifactId>slf4j-log4j12</artifactId>
80
      <artifactId>slf4j-log4j12</artifactId>
96
      <scope>test</scope>
81
      <scope>test</scope>
97
    </dependency>
82
    </dependency>
98

    
   

   
99
    <dependency>

   
100
      <groupId>org.apache.hadoop</groupId>

   
101
      <artifactId>${hadoop.common.artifact.id}</artifactId>

   
102
      <optional>true</optional>

   
103
    </dependency>

   
104

    
   

   
105
    <dependency>
83
    <dependency>
106
      <groupId>com.google.protobuf</groupId>
84
      <groupId>com.google.protobuf</groupId>
107
      <artifactId>protobuf-java</artifactId>
85
      <artifactId>protobuf-java</artifactId>
108
      <scope>compile</scope>
86
      <scope>compile</scope>
109
    </dependency>
87
    </dependency>
110

    
   

   
111
  </dependencies>
88
  </dependencies>
112

    
   

   
113
  <profiles>
89
  <profiles>
114

    
   

   
115
    <profile>

   
116
      <id>hadoop-2</id>

   
117
      <activation>

   
118
        <property>

   
119
          <name>hadoop.profile</name>

   
120
          <value>2</value>

   
121
        </property>

   
122
      </activation>

   
123
      <dependencies>

   
124
        <dependency>

   
125
          <groupId>org.apache.hadoop</groupId>

   
126
          <artifactId>hadoop-auth</artifactId>

   
127
          <optional>true</optional>

   
128
        </dependency>

   
129
      </dependencies>

   
130
    </profile>

   
131

    
   

   
132

    
   

   
133
   <profile>
90
    <profile>
134
     <id>compile-proto</id>
91
      <id>compile-proto</id>
135
     <build>
92
      <build>
136
      <plugins>
93
        <plugins>
137
        <plugin>
94
          <plugin>
[+20] [20] 41 lines
[+20]
179
          </executions>
136
            </executions>
180
        </plugin>
137
          </plugin>
181
      </plugins>
138
        </plugins>
182
     </build>
139
      </build>
183
   </profile>
140
    </profile>
184

    
   

   
185
  </profiles>
141
  </profiles>
186

    
   

   
187

    
   

   
188

    
   

   
189
</project>
142
</project>
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FlumeEvent.java
Revision c447335 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java
Revision 073042f New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Writable.java
New File
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/WritableUtils.java
New File
 
flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
Revision 563dbcc New Change
 
  1. flume-ng-channels/flume-file-channel/pom.xml: Loading...
  2. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FlumeEvent.java: Loading...
  3. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java: Loading...
  4. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Writable.java: Loading...
  5. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/WritableUtils.java: Loading...
  6. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java: Loading...