001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one or more
003     * contributor license agreements.  See the NOTICE file distributed with
004     * this work for additional information regarding copyright ownership.
005     * The ASF licenses this file to You under the Apache License, Version 2.0
006     * (the "License"); you may not use this file except in compliance with
007     * the License.  You may obtain a copy of the License at
008     *
009     *      http://www.apache.org/licenses/LICENSE-2.0
010     *
011     * Unless required by applicable law or agreed to in writing, software
012     * distributed under the License is distributed on an "AS IS" BASIS,
013     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014     * See the License for the specific language governing permissions and
015     * limitations under the License.
016     */
017    package org.apache.camel.component.file;
018    
019    import java.io.File;
020    import java.util.concurrent.ConcurrentHashMap;
021    
022    import org.apache.camel.AsyncCallback;
023    import org.apache.camel.Processor;
024    import org.apache.camel.impl.ScheduledPollConsumer;
025    import org.apache.commons.logging.Log;
026    import org.apache.commons.logging.LogFactory;
027    
028    /**
029     * For consuming files.
030     *
031     * @version $Revision: 41272 $
032     */
033    public class FileConsumer extends ScheduledPollConsumer<FileExchange> {
034        private static final transient Log LOG = LogFactory.getLog(FileConsumer.class);
035    
036        private FileEndpoint endpoint;
037        private ConcurrentHashMap<File, File> filesBeingProcessed = new ConcurrentHashMap<File, File>();
038        private ConcurrentHashMap<File, Long> fileSizes = new ConcurrentHashMap<File, Long>();
039        private ConcurrentHashMap<File, Long> noopMap = new ConcurrentHashMap<File, Long>();
040    
041        private boolean generateEmptyExchangeWhenIdle;
042        private boolean recursive = true;
043        private String regexPattern = "";
044    
045        private long lastPollTime;
046        private int unchangedDelay;
047        private boolean unchangedSize;
048    
049    
050        public FileConsumer(final FileEndpoint endpoint, Processor processor) {
051            super(endpoint, processor);
052            this.endpoint = endpoint;
053        }
054    
055        protected synchronized void poll() throws Exception {
056            int rc = pollFileOrDirectory(endpoint.getFile(), isRecursive());
057            if (rc == 0 && generateEmptyExchangeWhenIdle) {
058                final FileExchange exchange = endpoint.createExchange((File)null);
059                getAsyncProcessor().process(exchange, new AsyncCallback() {
060                    public void done(boolean sync) {
061                    }
062                });
063            }
064            lastPollTime = System.currentTimeMillis();
065        }
066    
067        /**
068         * Pools the given file or directory for files to process.
069         *
070         * @param fileOrDirectory  file or directory
071         * @param processDir  recursive
072         * @return the number of files processed or being processed async.
073         */
074        protected int pollFileOrDirectory(File fileOrDirectory, boolean processDir) {
075            if (!fileOrDirectory.isDirectory()) {
076                return pollFile(fileOrDirectory); // process the file
077            } else if (processDir) {
078                int rc = 0;
079                if (isValidFile(fileOrDirectory)) {
080                    LOG.debug("Polling directory " + fileOrDirectory);
081                    File[] files = fileOrDirectory.listFiles();
082                    for (File file : files) {
083                        rc += pollFileOrDirectory(file, isRecursive()); // self-recursion
084                    }
085                }
086                return rc;
087            } else {
088                LOG.debug("Skipping directory " + fileOrDirectory);
089                return 0;
090            }
091        }
092    
093        /**
094         * Polls the given file
095         *
096         * @param file  the file
097         * @return returns 1 if the file was processed, 0 otherwise.
098         */
099        protected int pollFile(final File file) {
100    
101            if (!file.exists()) {
102                return 0;
103            }
104            if (!isValidFile(file)) {
105                return 0;
106            }
107            // we only care about file modified times if we are not deleting/moving
108            // files
109            if (!endpoint.isNoop()) {
110                if (filesBeingProcessed.contains(file)) {
111                    return 1;
112                }
113                filesBeingProcessed.put(file, file);
114            }
115    
116            final FileProcessStrategy processStrategy = endpoint.getFileStrategy();
117            final FileExchange exchange = endpoint.createExchange(file);
118    
119            endpoint.configureMessage(file, exchange.getIn());
120            try {
121                if (LOG.isDebugEnabled()) {
122                    LOG.debug("About to process file:  " + file + " using exchange: " + exchange);
123                }
124                if (processStrategy.begin(endpoint, exchange, file)) {
125    
126                    // Use the async processor interface so that processing of
127                    // the exchange can happen asynchronously
128                    getAsyncProcessor().process(exchange, new AsyncCallback() {
129                        public void done(boolean sync) {
130                            if (exchange.getException() == null) {
131                                try {
132                                    processStrategy.commit(endpoint, exchange, file);
133                                } catch (Exception e) {
134                                    handleException(e);
135                                }
136                            } else {
137                                handleException(exchange.getException());
138                            }
139                            filesBeingProcessed.remove(file);
140                        }
141                    });
142    
143                } else {
144                    if (LOG.isDebugEnabled()) {
145                        LOG.debug(endpoint + " cannot process file: " + file);
146                    }
147                }
148            } catch (Throwable e) {
149                handleException(e);
150            }
151            return 1;
152        }
153    
154        protected boolean isValidFile(File file) {
155            boolean result = false;
156            if (file != null && file.exists()) {
157                // TODO: maybe use a configurable strategy instead of the hardcoded one based on last file change
158                if (isMatched(file) && isChanged(file)) {
159                    result = true;
160                }
161            }
162            return result;
163        }
164    
165    
166        protected boolean isChanged(File file) {
167            if (file == null) {
168                // Sanity check
169                return false;
170            } else if (file.isDirectory()) {
171                // Allow recursive polling to descend into this directory
172                return true;
173            } else {
174                boolean lastModifiedCheck = false;
175                long modifiedDuration = 0;
176                if (getUnchangedDelay() > 0) {
177                    modifiedDuration = System.currentTimeMillis() - file.lastModified();
178                    lastModifiedCheck = modifiedDuration >= getUnchangedDelay();
179                }
180    
181                long fileModified = file.lastModified();
182                Long previousModified = noopMap.get(file);
183                noopMap.put(file, fileModified);
184                if (previousModified == null || fileModified > previousModified) {
185                    lastModifiedCheck = true;
186                }
187    
188                boolean sizeCheck = false;
189                long sizeDifference = 0;
190                if (isUnchangedSize()) {
191                    Long value = fileSizes.get(file);
192                    if (value == null) {
193                        sizeCheck = true;
194                    } else {
195                        sizeCheck = file.length() != value;
196                    }
197                }
198    
199                boolean answer = lastModifiedCheck || sizeCheck;
200    
201                if (LOG.isDebugEnabled()) {
202                    LOG.debug("file:" + file + " isChanged:" + answer + " " + "sizeCheck:" + sizeCheck + "("
203                              + sizeDifference + ") " + "lastModifiedCheck:" + lastModifiedCheck + "("
204                              + modifiedDuration + ")");
205                }
206    
207                if (isUnchangedSize()) {
208                    if (answer) {
209                        fileSizes.put(file, file.length());
210                    } else {
211                        fileSizes.remove(file);
212                    }
213                }
214    
215                return answer;
216            }
217        }
218    
219        protected boolean isMatched(File file) {
220            String name = file.getName();
221            if (regexPattern != null && regexPattern.length() > 0) {
222                if (!name.matches(getRegexPattern())) {
223                    return false;
224                }
225            }
226            String[] prefixes = endpoint.getExcludedNamePrefixes();
227            if (prefixes != null) {
228                for (String prefix : prefixes) {
229                    if (name.startsWith(prefix)) {
230                        return false;
231                    }
232                }
233            }
234            String[] postfixes = endpoint.getExcludedNamePostfixes();
235            if (postfixes != null) {
236                for (String postfix : postfixes) {
237                    if (name.endsWith(postfix)) {
238                        return false;
239                    }
240                }
241            }
242            return true;
243        }
244    
245        public boolean isRecursive() {
246            return this.recursive;
247        }
248    
249        public void setRecursive(boolean recursive) {
250            this.recursive = recursive;
251        }
252    
253        public String getRegexPattern() {
254            return this.regexPattern;
255        }
256    
257        public void setRegexPattern(String regexPattern) {
258            this.regexPattern = regexPattern;
259        }
260    
261        public boolean isGenerateEmptyExchangeWhenIdle() {
262            return generateEmptyExchangeWhenIdle;
263        }
264    
265        public void setGenerateEmptyExchangeWhenIdle(boolean generateEmptyExchangeWhenIdle) {
266            this.generateEmptyExchangeWhenIdle = generateEmptyExchangeWhenIdle;
267        }
268    
269        public int getUnchangedDelay() {
270            return unchangedDelay;
271        }
272    
273        public void setUnchangedDelay(int unchangedDelay) {
274            this.unchangedDelay = unchangedDelay;
275        }
276    
277        public boolean isUnchangedSize() {
278            return unchangedSize;
279        }
280    
281        public void setUnchangedSize(boolean unchangedSize) {
282            this.unchangedSize = unchangedSize;
283        }
284    
285    }