Review Board 1.7.22


FLUME-1516. Write dual checkpoints.

Review Request #8899 - Created Jan. 9, 2013 and submitted

Hari Shreedharan
FLUME-1516
Reviewers
Flume
flume-git
Added support for backup and retrieval of checkpoint.
Added new unit tests. Current tests pass.

Diff revision 7

This is not the most recent revision of the diff. The latest diff is revision 11. See what's changed.

1 2 3 4 5 6 7 8 9 10 11
1 2 3 4 5 6 7 8 9 10 11

  1. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/EventQueueBackingStore.java: Loading...
  2. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/EventQueueBackingStoreFactory.java: Loading...
  3. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/EventQueueBackingStoreFile.java: Loading...
  4. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/EventQueueBackingStoreFileV3.java: Loading...
  5. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java: Loading...
  6. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannelConfiguration.java: Loading...
  7. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FlumeEventQueue.java: Loading...
  8. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java: Loading...
  9. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java: Loading...
  10. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java: Loading...
  11. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java: Loading...
  12. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java: Loading...
  13. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java: Loading...
  14. flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto: Loading...
  15. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannelBase.java: Loading...
  16. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannelRestart.java: Loading...
  17. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java: Loading...
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/EventQueueBackingStore.java
Revision b136eb0 New Change
[20] 15 lines
[+20]
16
 * specific language governing permissions and limitations
16
 * specific language governing permissions and limitations
17
 * under the License.
17
 * under the License.
18
 */
18
 */
19
package org.apache.flume.channel.file;
19
package org.apache.flume.channel.file;
20

    
   
20

   

    
   
21
import java.io.File;
21
import java.io.IOException;
22
import java.io.IOException;
22

    
   
23

   
23
import com.google.common.collect.ImmutableSortedSet;
24
import com.google.common.collect.ImmutableSortedSet;
24

    
   
25

   
25
abstract class EventQueueBackingStore {
26
abstract class EventQueueBackingStore {
26
  protected static final int EMPTY = 0;
27
  protected static final int EMPTY = 0;
27
  private int queueSize;
28
  private int queueSize;
28
  private int queueHead;
29
  private int queueHead;
29
  private long logWriteOrderID;
30
  private long logWriteOrderID;
30
  private final int capacity;
31
  private final int capacity;
31
  private final String name;
32
  private final String name;

    
   
33
  public static final String BACKUP_COMPLETE_FILENAME = "backupComplete";
32

    
   
34

   
33
  protected EventQueueBackingStore(int capacity, String name) {
35
  protected EventQueueBackingStore(int capacity, String name) {
34
    this.capacity = capacity;
36
    this.capacity = capacity;
35
    this.name = name;
37
    this.name = name;
36
  }
38
  }
[+20] [20] 5 lines
[+20]
42
  abstract void decrementFileID(int fileID);
44
  abstract void decrementFileID(int fileID);
43
  abstract ImmutableSortedSet<Integer> getReferenceCounts();
45
  abstract ImmutableSortedSet<Integer> getReferenceCounts();
44
  abstract long get(int index);
46
  abstract long get(int index);
45
  abstract void put(int index, long value);
47
  abstract void put(int index, long value);
46
  abstract boolean syncRequired();
48
  abstract boolean syncRequired();

    
   
49
  public static boolean backupExists(File backupDir) {

    
   
50
    return new File(backupDir, BACKUP_COMPLETE_FILENAME).exists();

    
   
51
  }
47
  abstract void close() throws IOException;
52
  abstract void close() throws IOException;
48

    
   
53

   
49
  protected abstract int getVersion();
54
  protected abstract int getVersion();
50

    
   
55

   
51
  int getSize() {
56
  int getSize() {
[+20] [20] 26 lines
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/EventQueueBackingStoreFactory.java
Revision a19bdb4 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/EventQueueBackingStoreFile.java
Revision 4115505 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/EventQueueBackingStoreFileV3.java
Revision 451a9d4 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java
Revision ff42d19 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannelConfiguration.java
Revision 24368b3 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FlumeEventQueue.java
Revision 1ed9547 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
Revision 6ffc824 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java
Revision 1db3717 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java
Revision f51935c New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java
Revision fa4fd9d New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java
Revision 7094d3c New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java
Revision e6d4957 New Change
 
flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto
Revision 3a4e828 New Change
 
flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannelBase.java
Revision 3da09ab New Change
 
flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannelRestart.java
Revision 170dc72 New Change
 
flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
Revision ba653e6 New Change
 
  1. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/EventQueueBackingStore.java: Loading...
  2. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/EventQueueBackingStoreFactory.java: Loading...
  3. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/EventQueueBackingStoreFile.java: Loading...
  4. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/EventQueueBackingStoreFileV3.java: Loading...
  5. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java: Loading...
  6. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannelConfiguration.java: Loading...
  7. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FlumeEventQueue.java: Loading...
  8. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java: Loading...
  9. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java: Loading...
  10. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java: Loading...
  11. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java: Loading...
  12. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java: Loading...
  13. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java: Loading...
  14. flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto: Loading...
  15. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannelBase.java: Loading...
  16. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannelRestart.java: Loading...
  17. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java: Loading...