Review Board 1.7.22


FLUME-1632: Persist progress on each file in file spooling client/source

Review Request #8596 - Created Dec. 14, 2012 and submitted

Mike Percy
FLUME-1632
Reviewers
Flume
flume-git
Defines EventDeserializer interface and uses it from the spooling source. Progress is persisted as bytes are read from the underlying file.
Unit tests pass.

Diff revision 3

This is not the most recent revision of the diff. The latest diff is revision 5. See what's changed.

1 2 3 4 5
1 2 3 4 5

  1. flume-ng-core/src/main/java/org/apache/flume/serialization/PositionTracker.java: Loading...
  2. flume-ng-core/src/main/java/org/apache/flume/serialization/Resettable.java: Loading...
  3. flume-ng-core/src/main/java/org/apache/flume/serialization/ResettableFileInputStream.java: Loading...
  4. flume-ng-core/src/main/java/org/apache/flume/serialization/ResettableInputStream.java: Loading...
  5. flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySource.java: Loading...
  6. flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySourceConfigurationConstants.java: Loading...
  7. flume-ng-core/src/main/java/org/apache/flume/tools/PlatformDetect.java: Loading...
  8. flume-ng-core/src/test/java/org/apache/flume/client/avro/TestBufferedLineReader.java: Loading...
  9. flume-ng-core/src/test/java/org/apache/flume/client/avro/TestSpoolingFileLineReader.java: Loading...
  10. flume-ng-core/src/test/java/org/apache/flume/serialization/ResettableTestStringInputStream.java: Loading...
  11. flume-ng-core/src/test/java/org/apache/flume/serialization/TestDurablePositionTracker.java: Loading...
  12. flume-ng-core/src/test/java/org/apache/flume/serialization/TestLineDeserializer.java: Loading...
  13. flume-ng-core/src/test/java/org/apache/flume/serialization/TestResettableFileInputStream.java: Loading...
  14. flume-ng-core/src/test/java/org/apache/flume/serialization/TestSpoolingFileEventParser.java: Loading...
  15. flume-ng-core/src/test/java/org/apache/flume/source/TestSpoolDirectorySource.java: Loading...
  16. flume-ng-core/src/test/resources/TestResettableFileInputStream_1.avro: Loading...
  17. flume-ng-core/src/test/resources/TestResettableFileInputStream_1.truncated.avro: Loading...
This diff has been split across 2 pages: < 1 2
flume-ng-core/src/main/java/org/apache/flume/serialization/PositionTracker.java
New File

    
   
1
/**

    
   
2
 * Licensed to the Apache Software Foundation (ASF) under one

    
   
3
 * or more contributor license agreements.  See the NOTICE file

    
   
4
 * distributed with this work for additional information

    
   
5
 * regarding copyright ownership.  The ASF licenses this file

    
   
6
 * to you under the Apache License, Version 2.0 (the

    
   
7
 * "License"); you may not use this file except in compliance

    
   
8
 * with the License.  You may obtain a copy of the License at

    
   
9
 *

    
   
10
 *     http://www.apache.org/licenses/LICENSE-2.0

    
   
11
 *

    
   
12
 * Unless required by applicable law or agreed to in writing, software

    
   
13
 * distributed under the License is distributed on an "AS IS" BASIS,

    
   
14
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

    
   
15
 * See the License for the specific language governing permissions and

    
   
16
 * limitations under the License.

    
   
17
 */

    
   
18
package org.apache.flume.serialization;

    
   
19

   

    
   
20
import org.apache.flume.annotations.InterfaceAudience;

    
   
21
import org.apache.flume.annotations.InterfaceStability;

    
   
22

   

    
   
23
import java.io.Closeable;

    
   
24
import java.io.IOException;

    
   
25

   

    
   
26
/**

    
   
27
 * Defines an interface for tracking the offset position in a target file.

    
   
28
 */

    
   
29
@InterfaceAudience.Private

    
   
30
@InterfaceStability.Evolving

    
   
31
public interface PositionTracker extends Closeable {

    
   
32
  void storePosition(long position) throws IOException;

    
   
33
  long getPosition();

    
   
34
  String getTarget();

    
   
35
  void close() throws IOException;

    
   
36
}
flume-ng-core/src/main/java/org/apache/flume/serialization/Resettable.java
New File
 
flume-ng-core/src/main/java/org/apache/flume/serialization/ResettableFileInputStream.java
New File
 
flume-ng-core/src/main/java/org/apache/flume/serialization/ResettableInputStream.java
New File
 
flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySource.java
Revision 61824d8 New Change
 
flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySourceConfigurationConstants.java
Revision 806a661 New Change
 
flume-ng-core/src/main/java/org/apache/flume/tools/PlatformDetect.java
New File
 
flume-ng-core/src/test/java/org/apache/flume/client/avro/TestBufferedLineReader.java
Revision 169abe5 New Change
 
flume-ng-core/src/test/java/org/apache/flume/client/avro/TestSpoolingFileLineReader.java
Revision 740bc98 New Change
 
flume-ng-core/src/test/java/org/apache/flume/serialization/ResettableTestStringInputStream.java
New File
 
flume-ng-core/src/test/java/org/apache/flume/serialization/TestDurablePositionTracker.java
New File
 
flume-ng-core/src/test/java/org/apache/flume/serialization/TestLineDeserializer.java
New File
 
flume-ng-core/src/test/java/org/apache/flume/serialization/TestResettableFileInputStream.java
New File
 
flume-ng-core/src/test/java/org/apache/flume/serialization/TestSpoolingFileEventParser.java
New File
 
flume-ng-core/src/test/java/org/apache/flume/source/TestSpoolDirectorySource.java
Revision 6e87b21 New Change
 
flume-ng-core/src/test/resources/TestResettableFileInputStream_1.avro
New File
 
flume-ng-core/src/test/resources/TestResettableFileInputStream_1.truncated.avro
New File
 
  1. flume-ng-core/src/main/java/org/apache/flume/serialization/PositionTracker.java: Loading...
  2. flume-ng-core/src/main/java/org/apache/flume/serialization/Resettable.java: Loading...
  3. flume-ng-core/src/main/java/org/apache/flume/serialization/ResettableFileInputStream.java: Loading...
  4. flume-ng-core/src/main/java/org/apache/flume/serialization/ResettableInputStream.java: Loading...
  5. flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySource.java: Loading...
  6. flume-ng-core/src/main/java/org/apache/flume/source/SpoolDirectorySourceConfigurationConstants.java: Loading...
  7. flume-ng-core/src/main/java/org/apache/flume/tools/PlatformDetect.java: Loading...
  8. flume-ng-core/src/test/java/org/apache/flume/client/avro/TestBufferedLineReader.java: Loading...
  9. flume-ng-core/src/test/java/org/apache/flume/client/avro/TestSpoolingFileLineReader.java: Loading...
  10. flume-ng-core/src/test/java/org/apache/flume/serialization/ResettableTestStringInputStream.java: Loading...
  11. flume-ng-core/src/test/java/org/apache/flume/serialization/TestDurablePositionTracker.java: Loading...
  12. flume-ng-core/src/test/java/org/apache/flume/serialization/TestLineDeserializer.java: Loading...
  13. flume-ng-core/src/test/java/org/apache/flume/serialization/TestResettableFileInputStream.java: Loading...
  14. flume-ng-core/src/test/java/org/apache/flume/serialization/TestSpoolingFileEventParser.java: Loading...
  15. flume-ng-core/src/test/java/org/apache/flume/source/TestSpoolDirectorySource.java: Loading...
  16. flume-ng-core/src/test/resources/TestResettableFileInputStream_1.avro: Loading...
  17. flume-ng-core/src/test/resources/TestResettableFileInputStream_1.truncated.avro: Loading...
This diff has been split across 2 pages: < 1 2