001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.component.file;
018
019 import java.io.File;
020 import java.io.IOException;
021 import java.io.RandomAccessFile;
022 import java.nio.channels.FileChannel;
023 import java.nio.channels.FileLock;
024 import java.util.concurrent.ConcurrentHashMap;
025
026 import org.apache.camel.AsyncCallback;
027 import org.apache.camel.Processor;
028 import org.apache.camel.impl.ScheduledPollConsumer;
029 import org.apache.camel.processor.DeadLetterChannel;
030 import org.apache.camel.util.ObjectHelper;
031 import org.apache.commons.logging.Log;
032 import org.apache.commons.logging.LogFactory;
033
034 /**
035 * For consuming files.
036 *
037 * @version $Revision: 46133 $
038 */
039 public class FileConsumer extends ScheduledPollConsumer<FileExchange> {
040 private static final transient Log LOG = LogFactory.getLog(FileConsumer.class);
041
042 private FileEndpoint endpoint;
043 private ConcurrentHashMap<File, File> filesBeingProcessed = new ConcurrentHashMap<File, File>();
044 private ConcurrentHashMap<File, Long> fileSizes = new ConcurrentHashMap<File, Long>();
045 private ConcurrentHashMap<File, Long> noopMap = new ConcurrentHashMap<File, Long>();
046
047 private long lastPollTime;
048 private int unchangedDelay;
049 private boolean unchangedSize;
050
051 private boolean generateEmptyExchangeWhenIdle;
052 private boolean recursive;
053 private String regexPattern = "";
054 private boolean exclusiveReadLock = true;
055 private boolean alwaysConsume;
056
057 public FileConsumer(final FileEndpoint endpoint, Processor processor) {
058 super(endpoint, processor);
059 this.endpoint = endpoint;
060 }
061
062 protected synchronized void poll() throws Exception {
063 // should be true the first time as its the top directory
064 int rc = pollFileOrDirectory(endpoint.getFile(), true);
065
066 // if no files consumes and using generateEmptyExchangeWhenIdle option then process an empty exchange
067 if (rc == 0 && generateEmptyExchangeWhenIdle) {
068 final FileExchange exchange = endpoint.createExchange((File)null);
069 getAsyncProcessor().process(exchange, new AsyncCallback() {
070 public void done(boolean sync) {
071 }
072 });
073 }
074
075 lastPollTime = System.currentTimeMillis();
076 }
077
078 /**
079 * Pools the given file or directory for files to process.
080 *
081 * @param fileOrDirectory file or directory
082 * @param processDir recursive
083 * @return the number of files processed or being processed async.
084 */
085 protected int pollFileOrDirectory(File fileOrDirectory, boolean processDir) {
086 if (!fileOrDirectory.isDirectory()) {
087 // process the file
088 return pollFile(fileOrDirectory);
089 } else if (processDir) {
090 // directory that can be recursive
091 int rc = 0;
092 if (isValidFile(fileOrDirectory)) {
093 if (LOG.isTraceEnabled()) {
094 LOG.trace("Polling directory " + fileOrDirectory);
095 }
096 File[] files = fileOrDirectory.listFiles();
097 for (File file : files) {
098 rc += pollFileOrDirectory(file, isRecursive()); // self-recursion
099 }
100 }
101 return rc;
102 } else {
103 if (LOG.isTraceEnabled()) {
104 LOG.trace("Skipping directory " + fileOrDirectory);
105 }
106 return 0;
107 }
108 }
109
110 /**
111 * Polls the given file
112 *
113 * @param file the file
114 * @return returns 1 if the file was processed, 0 otherwise.
115 */
116 protected int pollFile(final File file) {
117 if (LOG.isTraceEnabled()) {
118 LOG.trace("Polling file: " + file);
119 }
120
121 if (!file.exists()) {
122 return 0;
123 }
124 if (!isValidFile(file)) {
125 return 0;
126 }
127 // we only care about file modified times if we are not deleting/moving files
128 if (!endpoint.isNoop()) {
129 if (filesBeingProcessed.contains(file)) {
130 return 1;
131 }
132 filesBeingProcessed.put(file, file);
133 }
134
135 final FileProcessStrategy processStrategy = endpoint.getFileStrategy();
136 final FileExchange exchange = endpoint.createExchange(file);
137
138 endpoint.configureMessage(file, exchange.getIn());
139 try {
140 // is we use excluse read then acquire the exclusive read (waiting until we got it)
141 if (exclusiveReadLock) {
142 acquireExclusiveReadLock(file);
143 }
144
145 if (LOG.isDebugEnabled()) {
146 LOG.debug("About to process file: " + file + " using exchange: " + exchange);
147 }
148 if (processStrategy.begin(endpoint, exchange, file)) {
149
150 // Use the async processor interface so that processing of
151 // the exchange can happen asynchronously
152 getAsyncProcessor().process(exchange, new AsyncCallback() {
153 public void done(boolean sync) {
154 boolean failed = exchange.isFailed();
155 boolean handled = DeadLetterChannel.isFailureHandled(exchange);
156
157 if (LOG.isDebugEnabled()) {
158 LOG.debug("Done processing file: " + file + ". Status is: " + (failed ? "failed: " + failed + ", handled by failure processor: " + handled : "processed OK"));
159 }
160
161 boolean committed = false;
162 try {
163 if (!failed || handled) {
164 // commit the file strategy if there was no failure or already handled by the DeadLetterChannel
165 processStrategyCommit(processStrategy, exchange, file, handled);
166 committed = true;
167 } else {
168 // there was an exception but it was not handled by the DeadLetterChannel
169 handleException(exchange.getException());
170 }
171 } finally {
172 if (!committed) {
173 processStrategyRollback(processStrategy, exchange, file);
174 }
175 filesBeingProcessed.remove(file);
176 }
177 }
178 });
179
180 } else {
181 if (LOG.isDebugEnabled()) {
182 LOG.debug(endpoint + " can not process file: " + file);
183 }
184 }
185 } catch (Throwable e) {
186 handleException(e);
187 }
188
189 return 1;
190 }
191
192 /**
193 * Acquires exclusive read lock to the given file. Will wait until the lock is granted.
194 * After granting the read lock it is realeased, we just want to make sure that when we start
195 * consuming the file its not currently in progress of being written by third party.
196 */
197 protected void acquireExclusiveReadLock(File file) throws IOException {
198 if (LOG.isTraceEnabled()) {
199 LOG.trace("Waiting for exclusive read lock to file: " + file);
200 }
201
202 // try to acquire rw lock on the file before we can consume it
203 FileChannel channel = new RandomAccessFile(file, "rw").getChannel();
204 try {
205 FileLock lock = channel.lock();
206 if (LOG.isTraceEnabled()) {
207 LOG.trace("Acquired exclusive read lock: " + lock + " to file: " + file);
208 }
209 // just release it now we dont want to hold it during the rest of the processing
210 lock.release();
211 } finally {
212 // must close channel
213 ObjectHelper.close(channel, "FileConsumer during acquiring of exclusive read lock", LOG);
214 }
215 }
216
217 /**
218 * Strategy when the file was processed and a commit should be executed.
219 *
220 * @param processStrategy the strategy to perform the commit
221 * @param exchange the exchange
222 * @param file the file processed
223 * @param failureHandled is <tt>false</tt> if the exchange was processed succesfully, <tt>true</tt> if
224 * an exception occured during processing but it was handled by the failure processor (usually the
225 * DeadLetterChannel).
226 */
227 protected void processStrategyCommit(FileProcessStrategy processStrategy, FileExchange exchange,
228 File file, boolean failureHandled) {
229 try {
230 if (LOG.isDebugEnabled()) {
231 LOG.debug("Committing file strategy: " + processStrategy + " for file: " + file + (failureHandled ? " that was handled by the failure processor." : ""));
232 }
233 processStrategy.commit(endpoint, exchange, file);
234 } catch (Exception e) {
235 LOG.warn("Error committing file strategy: " + processStrategy, e);
236 handleException(e);
237 }
238 }
239
240 /**
241 * Strategy when the file was not processed and a rollback should be executed.
242 *
243 * @param processStrategy the strategy to perform the commit
244 * @param exchange the exchange
245 * @param file the file processed
246 */
247 protected void processStrategyRollback(FileProcessStrategy processStrategy, FileExchange exchange, File file) {
248 if (LOG.isDebugEnabled()) {
249 LOG.debug("Rolling back file strategy: " + processStrategy + " for file: " + file);
250 }
251 processStrategy.rollback(endpoint, exchange, file);
252 }
253
254 protected boolean isValidFile(File file) {
255 boolean result = false;
256 if (file != null && file.exists()) {
257 // TODO: maybe use a configurable strategy instead of the hardcoded one based on last file change
258 if (isMatched(file) && (alwaysConsume || isChanged(file))) {
259 result = true;
260 }
261 }
262 return result;
263 }
264
265 protected boolean isChanged(File file) {
266 if (file == null) {
267 // Sanity check
268 return false;
269 } else if (file.isDirectory()) {
270 // Allow recursive polling to descend into this directory
271 return true;
272 } else {
273 boolean lastModifiedCheck = false;
274 long modifiedDuration = 0;
275 if (getUnchangedDelay() > 0) {
276 modifiedDuration = System.currentTimeMillis() - file.lastModified();
277 lastModifiedCheck = modifiedDuration >= getUnchangedDelay();
278 }
279
280 long fileModified = file.lastModified();
281 Long previousModified = noopMap.get(file);
282 noopMap.put(file, fileModified);
283 if (previousModified == null || fileModified > previousModified) {
284 lastModifiedCheck = true;
285 }
286
287 boolean sizeCheck = false;
288 long sizeDifference = 0;
289 if (isUnchangedSize()) {
290 Long value = fileSizes.get(file);
291 if (value == null) {
292 sizeCheck = true;
293 } else {
294 sizeCheck = file.length() != value;
295 }
296 }
297
298 boolean answer = lastModifiedCheck || sizeCheck;
299
300 if (LOG.isDebugEnabled()) {
301 LOG.debug("file:" + file + " isChanged:" + answer + " " + "sizeCheck:" + sizeCheck + "("
302 + sizeDifference + ") " + "lastModifiedCheck:" + lastModifiedCheck + "("
303 + modifiedDuration + ")");
304 }
305
306 if (isUnchangedSize()) {
307 if (answer) {
308 fileSizes.put(file, file.length());
309 } else {
310 fileSizes.remove(file);
311 }
312 }
313
314 return answer;
315 }
316 }
317
318 protected boolean isMatched(File file) {
319 String name = file.getName();
320
321 // folders/names starting with dot is always skipped (eg. ".", ".camel", ".camelLock")
322 if (name.startsWith(".")) {
323 return false;
324 }
325 // lock files should be skipped
326 if (name.endsWith(FileEndpoint.DEFAULT_LOCK_FILE_POSTFIX)) {
327 return false;
328 }
329
330 if (regexPattern != null && regexPattern.length() > 0) {
331 if (!name.matches(regexPattern)) {
332 return false;
333 }
334 }
335
336 if (endpoint.getExcludedNamePrefix() != null) {
337 if (name.startsWith(endpoint.getExcludedNamePrefix())) {
338 return false;
339 }
340 }
341 String[] prefixes = endpoint.getExcludedNamePrefixes();
342 if (prefixes != null) {
343 for (String prefix : prefixes) {
344 if (name.startsWith(prefix)) {
345 return false;
346 }
347 }
348 }
349 if (endpoint.getExcludedNamePostfix() != null) {
350 if (name.endsWith(endpoint.getExcludedNamePostfix())) {
351 return false;
352 }
353 }
354 String[] postfixes = endpoint.getExcludedNamePostfixes();
355 if (postfixes != null) {
356 for (String postfix : postfixes) {
357 if (name.endsWith(postfix)) {
358 return false;
359 }
360 }
361 }
362
363 return true;
364 }
365
366 public boolean isRecursive() {
367 return this.recursive;
368 }
369
370 public void setRecursive(boolean recursive) {
371 this.recursive = recursive;
372 }
373
374 public String getRegexPattern() {
375 return this.regexPattern;
376 }
377
378 public void setRegexPattern(String regexPattern) {
379 this.regexPattern = regexPattern;
380 }
381
382 public boolean isGenerateEmptyExchangeWhenIdle() {
383 return generateEmptyExchangeWhenIdle;
384 }
385
386 public void setGenerateEmptyExchangeWhenIdle(boolean generateEmptyExchangeWhenIdle) {
387 this.generateEmptyExchangeWhenIdle = generateEmptyExchangeWhenIdle;
388 }
389
390 public int getUnchangedDelay() {
391 return unchangedDelay;
392 }
393
394 /**
395 * @deprecated will be removed in Camel 2.0
396 */
397 public void setUnchangedDelay(int unchangedDelay) {
398 this.unchangedDelay = unchangedDelay;
399 }
400
401 public boolean isUnchangedSize() {
402 return unchangedSize;
403 }
404
405 /**
406 * @deprecated will be removed in Camel 2.0
407 */
408 public void setUnchangedSize(boolean unchangedSize) {
409 this.unchangedSize = unchangedSize;
410 }
411
412 public boolean isExclusiveReadLock() {
413 return exclusiveReadLock;
414 }
415
416 public void setExclusiveReadLock(boolean exclusiveReadLock) {
417 this.exclusiveReadLock = exclusiveReadLock;
418 }
419
420 public boolean isAlwaysConsume() {
421 return alwaysConsume;
422 }
423
424 public void setAlwaysConsume(boolean alwaysConsume) {
425 this.alwaysConsume = alwaysConsume;
426 }
427 }