001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one or more
003     * contributor license agreements.  See the NOTICE file distributed with
004     * this work for additional information regarding copyright ownership.
005     * The ASF licenses this file to You under the Apache License, Version 2.0
006     * (the "License"); you may not use this file except in compliance with
007     * the License.  You may obtain a copy of the License at
008     *
009     *      http://www.apache.org/licenses/LICENSE-2.0
010     *
011     * Unless required by applicable law or agreed to in writing, software
012     * distributed under the License is distributed on an "AS IS" BASIS,
013     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014     * See the License for the specific language governing permissions and
015     * limitations under the License.
016     */
017    package org.apache.camel.component.file;
018    
019    import java.io.File;
020    import java.io.IOException;
021    import java.io.RandomAccessFile;
022    import java.nio.channels.FileChannel;
023    import java.nio.channels.FileLock;
024    import java.util.concurrent.ConcurrentHashMap;
025    
026    import org.apache.camel.AsyncCallback;
027    import org.apache.camel.Processor;
028    import org.apache.camel.impl.ScheduledPollConsumer;
029    import org.apache.camel.processor.DeadLetterChannel;
030    import org.apache.camel.util.ObjectHelper;
031    import org.apache.commons.logging.Log;
032    import org.apache.commons.logging.LogFactory;
033    
034    /**
035     * For consuming files.
036     *
037     * @version $Revision: 46133 $
038     */
039    public class FileConsumer extends ScheduledPollConsumer<FileExchange> {
040        private static final transient Log LOG = LogFactory.getLog(FileConsumer.class);
041    
042        private FileEndpoint endpoint;
043        private ConcurrentHashMap<File, File> filesBeingProcessed = new ConcurrentHashMap<File, File>();
044        private ConcurrentHashMap<File, Long> fileSizes = new ConcurrentHashMap<File, Long>();
045        private ConcurrentHashMap<File, Long> noopMap = new ConcurrentHashMap<File, Long>();
046    
047        private long lastPollTime;
048        private int unchangedDelay;
049        private boolean unchangedSize;
050    
051        private boolean generateEmptyExchangeWhenIdle;
052        private boolean recursive;
053        private String regexPattern = "";
054        private boolean exclusiveReadLock = true;
055        private boolean alwaysConsume;
056    
057        public FileConsumer(final FileEndpoint endpoint, Processor processor) {
058            super(endpoint, processor);
059            this.endpoint = endpoint;
060        }
061    
062        protected synchronized void poll() throws Exception {
063            // should be true the first time as its the top directory
064            int rc = pollFileOrDirectory(endpoint.getFile(), true);
065    
066            // if no files consumes and using generateEmptyExchangeWhenIdle option then process an empty exchange 
067            if (rc == 0 && generateEmptyExchangeWhenIdle) {
068                final FileExchange exchange = endpoint.createExchange((File)null);
069                getAsyncProcessor().process(exchange, new AsyncCallback() {
070                    public void done(boolean sync) {
071                    }
072                });
073            }
074    
075            lastPollTime = System.currentTimeMillis();
076        }
077    
078        /**
079         * Pools the given file or directory for files to process.
080         *
081         * @param fileOrDirectory  file or directory
082         * @param processDir  recursive
083         * @return the number of files processed or being processed async.
084         */
085        protected int pollFileOrDirectory(File fileOrDirectory, boolean processDir) {
086            if (!fileOrDirectory.isDirectory()) {
087                // process the file
088                return pollFile(fileOrDirectory);
089            } else if (processDir) {
090                // directory that can be recursive
091                int rc = 0;
092                if (isValidFile(fileOrDirectory)) {
093                    if (LOG.isTraceEnabled()) {
094                        LOG.trace("Polling directory " + fileOrDirectory);
095                    }
096                    File[] files = fileOrDirectory.listFiles();
097                    for (File file : files) {
098                        rc += pollFileOrDirectory(file, isRecursive()); // self-recursion
099                    }
100                }
101                return rc;
102            } else {
103                if (LOG.isTraceEnabled()) {
104                    LOG.trace("Skipping directory " + fileOrDirectory);
105                }
106                return 0;
107            }
108        }
109    
110        /**
111         * Polls the given file
112         *
113         * @param file  the file
114         * @return returns 1 if the file was processed, 0 otherwise.
115         */
116        protected int pollFile(final File file) {
117            if (LOG.isTraceEnabled()) {
118                LOG.trace("Polling file: " + file);
119            }
120    
121            if (!file.exists()) {
122                return 0;
123            }
124            if (!isValidFile(file)) {
125                return 0;
126            }
127            // we only care about file modified times if we are not deleting/moving files
128            if (!endpoint.isNoop()) {
129                if (filesBeingProcessed.contains(file)) {
130                    return 1;
131                }
132                filesBeingProcessed.put(file, file);
133            }
134    
135            final FileProcessStrategy processStrategy = endpoint.getFileStrategy();
136            final FileExchange exchange = endpoint.createExchange(file);
137    
138            endpoint.configureMessage(file, exchange.getIn());
139            try {
140                // is we use excluse read then acquire the exclusive read (waiting until we got it)
141                if (exclusiveReadLock) {
142                    acquireExclusiveReadLock(file);
143                }
144    
145                if (LOG.isDebugEnabled()) {
146                    LOG.debug("About to process file: " + file + " using exchange: " + exchange);
147                }
148                if (processStrategy.begin(endpoint, exchange, file)) {
149    
150                    // Use the async processor interface so that processing of
151                    // the exchange can happen asynchronously
152                    getAsyncProcessor().process(exchange, new AsyncCallback() {
153                        public void done(boolean sync) {
154                            boolean failed = exchange.isFailed();
155                            boolean handled = DeadLetterChannel.isFailureHandled(exchange);
156    
157                            if (LOG.isDebugEnabled()) {
158                                LOG.debug("Done processing file: " + file + ". Status is: " + (failed ? "failed: " + failed + ", handled by failure processor: " + handled : "processed OK"));
159                            }
160    
161                            boolean committed = false;
162                            try {
163                                if (!failed || handled) {
164                                    // commit the file strategy if there was no failure or already handled by the DeadLetterChannel
165                                    processStrategyCommit(processStrategy, exchange, file, handled);
166                                    committed = true;
167                                } else {
168                                    // there was an exception but it was not handled by the DeadLetterChannel
169                                    handleException(exchange.getException());
170                                }
171                            } finally {
172                                if (!committed) {
173                                    processStrategyRollback(processStrategy, exchange, file);
174                                }
175                                filesBeingProcessed.remove(file);
176                            }
177                        }
178                    });
179    
180                } else {
181                    if (LOG.isDebugEnabled()) {
182                        LOG.debug(endpoint + " can not process file: " + file);
183                    }
184                }
185            } catch (Throwable e) {
186                handleException(e);
187            }
188    
189            return 1;
190        }
191    
192        /**
193         * Acquires exclusive read lock to the given file. Will wait until the lock is granted.
194         * After granting the read lock it is realeased, we just want to make sure that when we start
195         * consuming the file its not currently in progress of being written by third party.
196         */
197        protected void acquireExclusiveReadLock(File file) throws IOException {
198            if (LOG.isTraceEnabled()) {
199                LOG.trace("Waiting for exclusive read lock to file: " + file);
200            }
201    
202            // try to acquire rw lock on the file before we can consume it
203            FileChannel channel = new RandomAccessFile(file, "rw").getChannel();
204            try {
205                FileLock lock = channel.lock();
206                if (LOG.isTraceEnabled()) {
207                    LOG.trace("Acquired exclusive read lock: " + lock + " to file: " + file);
208                }
209                // just release it now we dont want to hold it during the rest of the processing
210                lock.release();
211            } finally {
212                // must close channel
213                ObjectHelper.close(channel, "FileConsumer during acquiring of exclusive read lock", LOG);
214            }
215        }
216    
217        /**
218         * Strategy when the file was processed and a commit should be executed.
219         *
220         * @param processStrategy   the strategy to perform the commit
221         * @param exchange          the exchange
222         * @param file              the file processed
223         * @param failureHandled    is <tt>false</tt> if the exchange was processed succesfully, <tt>true</tt> if
224         * an exception occured during processing but it was handled by the failure processor (usually the
225         * DeadLetterChannel).
226         */
227        protected void processStrategyCommit(FileProcessStrategy processStrategy, FileExchange exchange,
228                                             File file, boolean failureHandled) {
229            try {
230                if (LOG.isDebugEnabled()) {
231                    LOG.debug("Committing file strategy: " + processStrategy + " for file: " + file + (failureHandled ? " that was handled by the failure processor." : ""));
232                }
233                processStrategy.commit(endpoint, exchange, file);
234            } catch (Exception e) {
235                LOG.warn("Error committing file strategy: " + processStrategy, e);
236                handleException(e);
237            }
238        }
239    
240        /**
241         * Strategy when the file was not processed and a rollback should be executed.
242         *
243         * @param processStrategy   the strategy to perform the commit
244         * @param exchange          the exchange
245         * @param file              the file processed
246         */
247        protected void processStrategyRollback(FileProcessStrategy processStrategy, FileExchange exchange, File file) {
248            if (LOG.isDebugEnabled()) {
249                LOG.debug("Rolling back file strategy: " + processStrategy + " for file: " + file);
250            }
251            processStrategy.rollback(endpoint, exchange, file);
252        }
253    
254        protected boolean isValidFile(File file) {
255            boolean result = false;
256            if (file != null && file.exists()) {
257                // TODO: maybe use a configurable strategy instead of the hardcoded one based on last file change
258                if (isMatched(file) && (alwaysConsume || isChanged(file))) {
259                    result = true;
260                }
261            }
262            return result;
263        }
264    
265        protected boolean isChanged(File file) {
266            if (file == null) {
267                // Sanity check
268                return false;
269            } else if (file.isDirectory()) {
270                // Allow recursive polling to descend into this directory
271                return true;
272            } else {
273                boolean lastModifiedCheck = false;
274                long modifiedDuration = 0;
275                if (getUnchangedDelay() > 0) {
276                    modifiedDuration = System.currentTimeMillis() - file.lastModified();
277                    lastModifiedCheck = modifiedDuration >= getUnchangedDelay();
278                }
279    
280                long fileModified = file.lastModified();
281                Long previousModified = noopMap.get(file);
282                noopMap.put(file, fileModified);
283                if (previousModified == null || fileModified > previousModified) {
284                    lastModifiedCheck = true;
285                }
286    
287                boolean sizeCheck = false;
288                long sizeDifference = 0;
289                if (isUnchangedSize()) {
290                    Long value = fileSizes.get(file);
291                    if (value == null) {
292                        sizeCheck = true;
293                    } else {
294                        sizeCheck = file.length() != value;
295                    }
296                }
297    
298                boolean answer = lastModifiedCheck || sizeCheck;
299    
300                if (LOG.isDebugEnabled()) {
301                    LOG.debug("file:" + file + " isChanged:" + answer + " " + "sizeCheck:" + sizeCheck + "("
302                              + sizeDifference + ") " + "lastModifiedCheck:" + lastModifiedCheck + "("
303                              + modifiedDuration + ")");
304                }
305    
306                if (isUnchangedSize()) {
307                    if (answer) {
308                        fileSizes.put(file, file.length());
309                    } else {
310                        fileSizes.remove(file);
311                    }
312                }
313    
314                return answer;
315            }
316        }
317    
318        protected boolean isMatched(File file) {
319            String name = file.getName();
320    
321            // folders/names starting with dot is always skipped (eg. ".", ".camel", ".camelLock")
322            if (name.startsWith(".")) {
323                return false;
324            }
325            // lock files should be skipped
326            if (name.endsWith(FileEndpoint.DEFAULT_LOCK_FILE_POSTFIX)) {
327                return false;
328            }
329    
330            if (regexPattern != null && regexPattern.length() > 0) {
331                if (!name.matches(regexPattern)) {
332                    return false;
333                }
334            }
335    
336            if (endpoint.getExcludedNamePrefix() != null) {
337                if (name.startsWith(endpoint.getExcludedNamePrefix())) {
338                    return false;
339                }
340            }
341            String[] prefixes = endpoint.getExcludedNamePrefixes();
342            if (prefixes != null) {
343                for (String prefix : prefixes) {
344                    if (name.startsWith(prefix)) {
345                        return false;
346                    }
347                }
348            }
349            if (endpoint.getExcludedNamePostfix() != null) {
350                if (name.endsWith(endpoint.getExcludedNamePostfix())) {
351                    return false;
352                }
353            }
354            String[] postfixes = endpoint.getExcludedNamePostfixes();
355            if (postfixes != null) {
356                for (String postfix : postfixes) {
357                    if (name.endsWith(postfix)) {
358                        return false;
359                    }
360                }
361            }
362    
363            return true;
364        }
365    
366        public boolean isRecursive() {
367            return this.recursive;
368        }
369    
370        public void setRecursive(boolean recursive) {
371            this.recursive = recursive;
372        }
373    
374        public String getRegexPattern() {
375            return this.regexPattern;
376        }
377    
378        public void setRegexPattern(String regexPattern) {
379            this.regexPattern = regexPattern;
380        }
381    
382        public boolean isGenerateEmptyExchangeWhenIdle() {
383            return generateEmptyExchangeWhenIdle;
384        }
385    
386        public void setGenerateEmptyExchangeWhenIdle(boolean generateEmptyExchangeWhenIdle) {
387            this.generateEmptyExchangeWhenIdle = generateEmptyExchangeWhenIdle;
388        }
389    
390        public int getUnchangedDelay() {
391            return unchangedDelay;
392        }
393    
394        /**
395         * @deprecated will be removed in Camel 2.0
396         */
397        public void setUnchangedDelay(int unchangedDelay) {
398            this.unchangedDelay = unchangedDelay;
399        }
400    
401        public boolean isUnchangedSize() {
402            return unchangedSize;
403        }
404    
405        /**
406         * @deprecated will be removed in Camel 2.0
407         */
408        public void setUnchangedSize(boolean unchangedSize) {
409            this.unchangedSize = unchangedSize;
410        }
411    
412        public boolean isExclusiveReadLock() {
413            return exclusiveReadLock;
414        }
415    
416        public void setExclusiveReadLock(boolean exclusiveReadLock) {
417            this.exclusiveReadLock = exclusiveReadLock;
418        }
419    
420        public boolean isAlwaysConsume() {
421            return alwaysConsume;
422        }
423    
424        public void setAlwaysConsume(boolean alwaysConsume) {
425            this.alwaysConsume = alwaysConsume;
426        }
427    }