Review Board 1.7.22


FLUME-1763: FileChannel Checkpoints should not be done without free space

Review Request #8379 - Created Dec. 6, 2012 and submitted

Brock Noland
trunk
FLUME-1763
Reviewers
Flume
flume-git
Adds a parameter which can be configured by defaults to 500MB. We will refuse to checkpoint or perform log writes in this case.
Two unit tests added.
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java
Revision 64a1350 New Change
[20] 76 lines
[+20] [+] public class FileChannel extends BasicChannelSemantics {
77
  private int capacity;
77
  private int capacity;
78
  private int keepAlive;
78
  private int keepAlive;
79
  private int transactionCapacity;
79
  private int transactionCapacity;
80
  private long checkpointInterval;
80
  private long checkpointInterval;
81
  private long maxFileSize;
81
  private long maxFileSize;

    
   
82
  private long minimumRequiredSpace;
82
  private File checkpointDir;
83
  private File checkpointDir;
83
  private File[] dataDirs;
84
  private File[] dataDirs;
84
  private Log log;
85
  private Log log;
85
  private volatile boolean open;
86
  private volatile boolean open;
86
  private volatile Throwable startupError;
87
  private volatile Throwable startupError;
[+20] [20] 85 lines
[+20] [+] public void configure(Context context) {
172
    maxFileSize = Math.min(
173
    maxFileSize = Math.min(
173
        context.getLong(FileChannelConfiguration.MAX_FILE_SIZE,
174
        context.getLong(FileChannelConfiguration.MAX_FILE_SIZE,
174
            FileChannelConfiguration.DEFAULT_MAX_FILE_SIZE),
175
            FileChannelConfiguration.DEFAULT_MAX_FILE_SIZE),
175
            FileChannelConfiguration.DEFAULT_MAX_FILE_SIZE);
176
            FileChannelConfiguration.DEFAULT_MAX_FILE_SIZE);
176

    
   
177

   

    
   
178
    minimumRequiredSpace = Math.min(

    
   
179
        context.getLong(FileChannelConfiguration.MINIMUM_REQUIRED_SPACE,

    
   
180
            FileChannelConfiguration.DEFAULT_MINIMUM_REQUIRED_SPACE),

    
   
181
            FileChannelConfiguration.FLOOR_MINIMUM_REQUIRED_SPACE);

    
   
182

   
177
    logWriteTimeout = context.getInteger(
183
    logWriteTimeout = context.getInteger(
178
        FileChannelConfiguration.LOG_WRITE_TIMEOUT,
184
        FileChannelConfiguration.LOG_WRITE_TIMEOUT,
179
        FileChannelConfiguration.DEFAULT_WRITE_TIMEOUT);
185
        FileChannelConfiguration.DEFAULT_WRITE_TIMEOUT);
180

    
   
186

   
181
    if (logWriteTimeout < 0) {
187
    if (logWriteTimeout < 0) {
[+20] [20] 74 lines
[+20] public void configure(Context context) {
256
    LOG.info("Starting {}...", this);
262
    LOG.info("Starting {}...", this);
257
    try {
263
    try {
258
      Builder builder = new Log.Builder();
264
      Builder builder = new Log.Builder();
259
      builder.setCheckpointInterval(checkpointInterval);
265
      builder.setCheckpointInterval(checkpointInterval);
260
      builder.setMaxFileSize(maxFileSize);
266
      builder.setMaxFileSize(maxFileSize);

    
   
267
      builder.setMinimumRequiredSpace(minimumRequiredSpace);
261
      builder.setQueueSize(capacity);
268
      builder.setQueueSize(capacity);
262
      builder.setLogWriteTimeout(logWriteTimeout);
269
      builder.setLogWriteTimeout(logWriteTimeout);
263
      builder.setCheckpointDir(checkpointDir);
270
      builder.setCheckpointDir(checkpointDir);
264
      builder.setLogDirs(dataDirs);
271
      builder.setLogDirs(dataDirs);
265
      builder.setChannelName(getName());
272
      builder.setChannelName(getName());
[+20] [20] 309 lines
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannelConfiguration.java
Revision 92cad77 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
Revision 6d1cf51 New Change
 
flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java
Revision a165d6a New Change
 
flume-ng-doc/sphinx/FlumeUserGuide.rst
Revision f8528bb New Change
 
  1. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java: Loading...
  2. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannelConfiguration.java: Loading...
  3. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java: Loading...
  4. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java: Loading...
  5. flume-ng-doc/sphinx/FlumeUserGuide.rst: Loading...