Review Board 1.7.22


FLUME-1699. Make the rename of the meta file platform neutral

Review Request #8087 - Created Nov. 16, 2012 and submitted

Hari Shreedharan
FLUME-1699
Reviewers
Flume
brocknoland
flume-git
If atomic rename fails, rename the current meta file to old, rename tmp to meta and then delete old.
All unit tests pass
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileFactory.java
Revision 82e14b0 New Change
[20] 106 lines
[+20] [+] static LogFile.RandomReader getRandomReader(File file,
107
      @Nullable KeyProvider encryptionKeyProvider)
107
      @Nullable KeyProvider encryptionKeyProvider)
108
      throws IOException {
108
      throws IOException {
109
    RandomAccessFile logFile = null;
109
    RandomAccessFile logFile = null;
110
    try {
110
    try {
111
      File metaDataFile = Serialization.getMetaDataFile(file);
111
      File metaDataFile = Serialization.getMetaDataFile(file);
112
      if(metaDataFile.exists()) {
112
      File oldMetadataFile = Serialization.getOldMetaDataFile(file);

    
   
113
      File tempMetadataFile = Serialization.getMetaDataTempFile(file);

    
   
114
      boolean hasMeta = false;

    
   
115
      // FLUME-1699:

    
   
116
      // If the platform does not support atomic rename, then we

    
   
117
      // renamed log.meta -> log.meta.old followed by log.meta.tmp -> log.meta

    
   
118
      // I am not sure if all platforms maintain file integrity during

    
   
119
      // file metadata update operations. So:

    
   
120
      // 1. check if meta file exists

    
   
121
      // 2. If 1 returns false, check if temp exists

    
   
122
      // 3. if 2 is also false (maybe the machine died during temp->meta,

    
   
123
      //    then check if old exists.

    
   
124
      // In the above, we assume that if a file exists, it's integrity is ok.

    
   
125
      if (metaDataFile.exists()) {

    
   
126
        hasMeta = true;

    
   
127
      } else if (tempMetadataFile.exists()) {

    
   
128
        if (tempMetadataFile.renameTo(metaDataFile)) {

    
   
129
          hasMeta = true;

    
   
130
        } else {

    
   
131
          throw new IOException("Renaming of " + tempMetadataFile.getName()

    
   
132
                  + " to " + metaDataFile.getName() + " failed");

    
   
133
        }

    
   
134
      } else if (oldMetadataFile.exists()) {

    
   
135
        if (oldMetadataFile.renameTo(metaDataFile)) {

    
   
136
          hasMeta = true;

    
   
137
        } else {

    
   
138
          throw new IOException("Renaming of " + oldMetadataFile.getName()

    
   
139
                  + " to " + metaDataFile.getName() + " failed");

    
   
140
        }

    
   
141
      }

    
   
142
      if (hasMeta) {

    
   
143
        // Now the metadata file has been found, delete old or temp files

    
   
144
        // so it does not interfere with normal operation.

    
   
145
        if(oldMetadataFile.exists()) {

    
   
146
          oldMetadataFile.delete();

    
   
147
        }

    
   
148
        if(tempMetadataFile.exists()) {

    
   
149
          tempMetadataFile.delete();

    
   
150
        }
113
        return new LogFileV3.SequentialReader(file, encryptionKeyProvider);
151
        return new LogFileV3.SequentialReader(file, encryptionKeyProvider);
114
      }
152
      }
115
      logFile = new RandomAccessFile(file, "r");
153
      logFile = new RandomAccessFile(file, "r");
116
      int version = logFile.readInt();
154
      int version = logFile.readInt();
117
      if(Serialization.VERSION_2 == version) {
155
      if(Serialization.VERSION_2 == version) {
[+20] [20] 15 lines
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java
Revision f768d23 New Change
 
flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java
Revision 6b0eeb3 New Change
 
flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java
Revision 4b69698 New Change
 
  1. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileFactory.java: Loading...
  2. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java: Loading...
  3. flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java: Loading...
  4. flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java: Loading...