001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 package org.apache.camel.component.file; 018 019 import java.io.File; 020 import java.io.IOException; 021 import java.io.RandomAccessFile; 022 import java.nio.channels.FileChannel; 023 import java.nio.channels.FileLock; 024 import java.util.concurrent.ConcurrentHashMap; 025 026 import org.apache.camel.AsyncCallback; 027 import org.apache.camel.Processor; 028 import org.apache.camel.impl.ScheduledPollConsumer; 029 import org.apache.camel.processor.DeadLetterChannel; 030 import org.apache.camel.util.ObjectHelper; 031 import org.apache.commons.logging.Log; 032 import org.apache.commons.logging.LogFactory; 033 034 /** 035 * For consuming files. 036 * 037 * @version $Revision: 46133 $ 038 */ 039 public class FileConsumer extends ScheduledPollConsumer<FileExchange> { 040 private static final transient Log LOG = LogFactory.getLog(FileConsumer.class); 041 042 private FileEndpoint endpoint; 043 private ConcurrentHashMap<File, File> filesBeingProcessed = new ConcurrentHashMap<File, File>(); 044 private ConcurrentHashMap<File, Long> fileSizes = new ConcurrentHashMap<File, Long>(); 045 private ConcurrentHashMap<File, Long> noopMap = new ConcurrentHashMap<File, Long>(); 046 047 private long lastPollTime; 048 private int unchangedDelay; 049 private boolean unchangedSize; 050 051 private boolean generateEmptyExchangeWhenIdle; 052 private boolean recursive; 053 private String regexPattern = ""; 054 private boolean exclusiveReadLock = true; 055 private boolean alwaysConsume; 056 057 public FileConsumer(final FileEndpoint endpoint, Processor processor) { 058 super(endpoint, processor); 059 this.endpoint = endpoint; 060 } 061 062 protected synchronized void poll() throws Exception { 063 // should be true the first time as its the top directory 064 int rc = pollFileOrDirectory(endpoint.getFile(), true); 065 066 // if no files consumes and using generateEmptyExchangeWhenIdle option then process an empty exchange 067 if (rc == 0 && generateEmptyExchangeWhenIdle) { 068 final FileExchange exchange = endpoint.createExchange((File)null); 069 getAsyncProcessor().process(exchange, new AsyncCallback() { 070 public void done(boolean sync) { 071 } 072 }); 073 } 074 075 lastPollTime = System.currentTimeMillis(); 076 } 077 078 /** 079 * Pools the given file or directory for files to process. 080 * 081 * @param fileOrDirectory file or directory 082 * @param processDir recursive 083 * @return the number of files processed or being processed async. 084 */ 085 protected int pollFileOrDirectory(File fileOrDirectory, boolean processDir) { 086 if (!fileOrDirectory.isDirectory()) { 087 // process the file 088 return pollFile(fileOrDirectory); 089 } else if (processDir) { 090 // directory that can be recursive 091 int rc = 0; 092 if (isValidFile(fileOrDirectory)) { 093 if (LOG.isTraceEnabled()) { 094 LOG.trace("Polling directory " + fileOrDirectory); 095 } 096 File[] files = fileOrDirectory.listFiles(); 097 for (File file : files) { 098 rc += pollFileOrDirectory(file, isRecursive()); // self-recursion 099 } 100 } 101 return rc; 102 } else { 103 if (LOG.isTraceEnabled()) { 104 LOG.trace("Skipping directory " + fileOrDirectory); 105 } 106 return 0; 107 } 108 } 109 110 /** 111 * Polls the given file 112 * 113 * @param file the file 114 * @return returns 1 if the file was processed, 0 otherwise. 115 */ 116 protected int pollFile(final File file) { 117 if (LOG.isTraceEnabled()) { 118 LOG.trace("Polling file: " + file); 119 } 120 121 if (!file.exists()) { 122 return 0; 123 } 124 if (!isValidFile(file)) { 125 return 0; 126 } 127 // we only care about file modified times if we are not deleting/moving files 128 if (!endpoint.isNoop()) { 129 if (filesBeingProcessed.contains(file)) { 130 return 1; 131 } 132 filesBeingProcessed.put(file, file); 133 } 134 135 final FileProcessStrategy processStrategy = endpoint.getFileStrategy(); 136 final FileExchange exchange = endpoint.createExchange(file); 137 138 endpoint.configureMessage(file, exchange.getIn()); 139 try { 140 // is we use excluse read then acquire the exclusive read (waiting until we got it) 141 if (exclusiveReadLock) { 142 acquireExclusiveReadLock(file); 143 } 144 145 if (LOG.isDebugEnabled()) { 146 LOG.debug("About to process file: " + file + " using exchange: " + exchange); 147 } 148 if (processStrategy.begin(endpoint, exchange, file)) { 149 150 // Use the async processor interface so that processing of 151 // the exchange can happen asynchronously 152 getAsyncProcessor().process(exchange, new AsyncCallback() { 153 public void done(boolean sync) { 154 boolean failed = exchange.isFailed(); 155 boolean handled = DeadLetterChannel.isFailureHandled(exchange); 156 157 if (LOG.isDebugEnabled()) { 158 LOG.debug("Done processing file: " + file + ". Status is: " + (failed ? "failed: " + failed + ", handled by failure processor: " + handled : "processed OK")); 159 } 160 161 boolean committed = false; 162 try { 163 if (!failed || handled) { 164 // commit the file strategy if there was no failure or already handled by the DeadLetterChannel 165 processStrategyCommit(processStrategy, exchange, file, handled); 166 committed = true; 167 } else { 168 // there was an exception but it was not handled by the DeadLetterChannel 169 handleException(exchange.getException()); 170 } 171 } finally { 172 if (!committed) { 173 processStrategyRollback(processStrategy, exchange, file); 174 } 175 filesBeingProcessed.remove(file); 176 } 177 } 178 }); 179 180 } else { 181 if (LOG.isDebugEnabled()) { 182 LOG.debug(endpoint + " can not process file: " + file); 183 } 184 } 185 } catch (Throwable e) { 186 handleException(e); 187 } 188 189 return 1; 190 } 191 192 /** 193 * Acquires exclusive read lock to the given file. Will wait until the lock is granted. 194 * After granting the read lock it is realeased, we just want to make sure that when we start 195 * consuming the file its not currently in progress of being written by third party. 196 */ 197 protected void acquireExclusiveReadLock(File file) throws IOException { 198 if (LOG.isTraceEnabled()) { 199 LOG.trace("Waiting for exclusive read lock to file: " + file); 200 } 201 202 // try to acquire rw lock on the file before we can consume it 203 FileChannel channel = new RandomAccessFile(file, "rw").getChannel(); 204 try { 205 FileLock lock = channel.lock(); 206 if (LOG.isTraceEnabled()) { 207 LOG.trace("Acquired exclusive read lock: " + lock + " to file: " + file); 208 } 209 // just release it now we dont want to hold it during the rest of the processing 210 lock.release(); 211 } finally { 212 // must close channel 213 ObjectHelper.close(channel, "FileConsumer during acquiring of exclusive read lock", LOG); 214 } 215 } 216 217 /** 218 * Strategy when the file was processed and a commit should be executed. 219 * 220 * @param processStrategy the strategy to perform the commit 221 * @param exchange the exchange 222 * @param file the file processed 223 * @param failureHandled is <tt>false</tt> if the exchange was processed succesfully, <tt>true</tt> if 224 * an exception occured during processing but it was handled by the failure processor (usually the 225 * DeadLetterChannel). 226 */ 227 protected void processStrategyCommit(FileProcessStrategy processStrategy, FileExchange exchange, 228 File file, boolean failureHandled) { 229 try { 230 if (LOG.isDebugEnabled()) { 231 LOG.debug("Committing file strategy: " + processStrategy + " for file: " + file + (failureHandled ? " that was handled by the failure processor." : "")); 232 } 233 processStrategy.commit(endpoint, exchange, file); 234 } catch (Exception e) { 235 LOG.warn("Error committing file strategy: " + processStrategy, e); 236 handleException(e); 237 } 238 } 239 240 /** 241 * Strategy when the file was not processed and a rollback should be executed. 242 * 243 * @param processStrategy the strategy to perform the commit 244 * @param exchange the exchange 245 * @param file the file processed 246 */ 247 protected void processStrategyRollback(FileProcessStrategy processStrategy, FileExchange exchange, File file) { 248 if (LOG.isDebugEnabled()) { 249 LOG.debug("Rolling back file strategy: " + processStrategy + " for file: " + file); 250 } 251 processStrategy.rollback(endpoint, exchange, file); 252 } 253 254 protected boolean isValidFile(File file) { 255 boolean result = false; 256 if (file != null && file.exists()) { 257 // TODO: maybe use a configurable strategy instead of the hardcoded one based on last file change 258 if (isMatched(file) && (alwaysConsume || isChanged(file))) { 259 result = true; 260 } 261 } 262 return result; 263 } 264 265 protected boolean isChanged(File file) { 266 if (file == null) { 267 // Sanity check 268 return false; 269 } else if (file.isDirectory()) { 270 // Allow recursive polling to descend into this directory 271 return true; 272 } else { 273 boolean lastModifiedCheck = false; 274 long modifiedDuration = 0; 275 if (getUnchangedDelay() > 0) { 276 modifiedDuration = System.currentTimeMillis() - file.lastModified(); 277 lastModifiedCheck = modifiedDuration >= getUnchangedDelay(); 278 } 279 280 long fileModified = file.lastModified(); 281 Long previousModified = noopMap.get(file); 282 noopMap.put(file, fileModified); 283 if (previousModified == null || fileModified > previousModified) { 284 lastModifiedCheck = true; 285 } 286 287 boolean sizeCheck = false; 288 long sizeDifference = 0; 289 if (isUnchangedSize()) { 290 Long value = fileSizes.get(file); 291 if (value == null) { 292 sizeCheck = true; 293 } else { 294 sizeCheck = file.length() != value; 295 } 296 } 297 298 boolean answer = lastModifiedCheck || sizeCheck; 299 300 if (LOG.isDebugEnabled()) { 301 LOG.debug("file:" + file + " isChanged:" + answer + " " + "sizeCheck:" + sizeCheck + "(" 302 + sizeDifference + ") " + "lastModifiedCheck:" + lastModifiedCheck + "(" 303 + modifiedDuration + ")"); 304 } 305 306 if (isUnchangedSize()) { 307 if (answer) { 308 fileSizes.put(file, file.length()); 309 } else { 310 fileSizes.remove(file); 311 } 312 } 313 314 return answer; 315 } 316 } 317 318 protected boolean isMatched(File file) { 319 String name = file.getName(); 320 321 // folders/names starting with dot is always skipped (eg. ".", ".camel", ".camelLock") 322 if (name.startsWith(".")) { 323 return false; 324 } 325 // lock files should be skipped 326 if (name.endsWith(FileEndpoint.DEFAULT_LOCK_FILE_POSTFIX)) { 327 return false; 328 } 329 330 if (regexPattern != null && regexPattern.length() > 0) { 331 if (!name.matches(regexPattern)) { 332 return false; 333 } 334 } 335 336 if (endpoint.getExcludedNamePrefix() != null) { 337 if (name.startsWith(endpoint.getExcludedNamePrefix())) { 338 return false; 339 } 340 } 341 String[] prefixes = endpoint.getExcludedNamePrefixes(); 342 if (prefixes != null) { 343 for (String prefix : prefixes) { 344 if (name.startsWith(prefix)) { 345 return false; 346 } 347 } 348 } 349 if (endpoint.getExcludedNamePostfix() != null) { 350 if (name.endsWith(endpoint.getExcludedNamePostfix())) { 351 return false; 352 } 353 } 354 String[] postfixes = endpoint.getExcludedNamePostfixes(); 355 if (postfixes != null) { 356 for (String postfix : postfixes) { 357 if (name.endsWith(postfix)) { 358 return false; 359 } 360 } 361 } 362 363 return true; 364 } 365 366 public boolean isRecursive() { 367 return this.recursive; 368 } 369 370 public void setRecursive(boolean recursive) { 371 this.recursive = recursive; 372 } 373 374 public String getRegexPattern() { 375 return this.regexPattern; 376 } 377 378 public void setRegexPattern(String regexPattern) { 379 this.regexPattern = regexPattern; 380 } 381 382 public boolean isGenerateEmptyExchangeWhenIdle() { 383 return generateEmptyExchangeWhenIdle; 384 } 385 386 public void setGenerateEmptyExchangeWhenIdle(boolean generateEmptyExchangeWhenIdle) { 387 this.generateEmptyExchangeWhenIdle = generateEmptyExchangeWhenIdle; 388 } 389 390 public int getUnchangedDelay() { 391 return unchangedDelay; 392 } 393 394 /** 395 * @deprecated will be removed in Camel 2.0 396 */ 397 public void setUnchangedDelay(int unchangedDelay) { 398 this.unchangedDelay = unchangedDelay; 399 } 400 401 public boolean isUnchangedSize() { 402 return unchangedSize; 403 } 404 405 /** 406 * @deprecated will be removed in Camel 2.0 407 */ 408 public void setUnchangedSize(boolean unchangedSize) { 409 this.unchangedSize = unchangedSize; 410 } 411 412 public boolean isExclusiveReadLock() { 413 return exclusiveReadLock; 414 } 415 416 public void setExclusiveReadLock(boolean exclusiveReadLock) { 417 this.exclusiveReadLock = exclusiveReadLock; 418 } 419 420 public boolean isAlwaysConsume() { 421 return alwaysConsume; 422 } 423 424 public void setAlwaysConsume(boolean alwaysConsume) { 425 this.alwaysConsume = alwaysConsume; 426 } 427 }