001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one or more
003     * contributor license agreements.  See the NOTICE file distributed with
004     * this work for additional information regarding copyright ownership.
005     * The ASF licenses this file to You under the Apache License, Version 2.0
006     * (the "License"); you may not use this file except in compliance with
007     * the License.  You may obtain a copy of the License at
008     *
009     *      http://www.apache.org/licenses/LICENSE-2.0
010     *
011     * Unless required by applicable law or agreed to in writing, software
012     * distributed under the License is distributed on an "AS IS" BASIS,
013     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014     * See the License for the specific language governing permissions and
015     * limitations under the License.
016     */
017    package org.apache.camel.component.file;
018    
019    import java.io.File;
020    import java.util.concurrent.ConcurrentHashMap;
021    
022    import org.apache.camel.AsyncCallback;
023    import org.apache.camel.Processor;
024    import org.apache.camel.impl.ScheduledPollConsumer;
025    import org.apache.commons.logging.Log;
026    import org.apache.commons.logging.LogFactory;
027    
028    /**
029     * For consuming files.
030     *
031     * @version $Revision: 37863 $
032     */
033    public class FileConsumer extends ScheduledPollConsumer<FileExchange> {
034        private static final transient Log LOG = LogFactory.getLog(FileConsumer.class);
035    
036        private FileEndpoint endpoint;
037        private ConcurrentHashMap<File, File> filesBeingProcessed = new ConcurrentHashMap<File, File>();
038        private ConcurrentHashMap<File, Long> fileSizes = new ConcurrentHashMap<File, Long>();
039    
040        private boolean generateEmptyExchangeWhenIdle;
041        private boolean recursive = true;
042        private String regexPattern = "";
043    
044        private long lastPollTime;
045        private int unchangedDelay;
046        private boolean unchangedSize;
047    
048        public FileConsumer(final FileEndpoint endpoint, Processor processor) {
049            super(endpoint, processor);
050            this.endpoint = endpoint;
051        }
052    
053        protected synchronized void poll() throws Exception {
054            int rc = pollFileOrDirectory(endpoint.getFile(), isRecursive());
055            if (rc == 0 && generateEmptyExchangeWhenIdle) {
056                final FileExchange exchange = endpoint.createExchange((File)null);
057                getAsyncProcessor().process(exchange, new AsyncCallback() {
058                    public void done(boolean sync) {
059                    }
060                });
061            }
062            lastPollTime = System.currentTimeMillis();
063        }
064    
065        /**
066         * Pools the given file or directory for files to process.
067         *
068         * @param fileOrDirectory  file or directory
069         * @param processDir  recursive
070         * @return the number of files processed or being processed async.
071         */
072        protected int pollFileOrDirectory(File fileOrDirectory, boolean processDir) {
073            if (!fileOrDirectory.isDirectory()) {
074                return pollFile(fileOrDirectory); // process the file
075            } else if (processDir) {
076                int rc = 0;
077                if (isValidFile(fileOrDirectory)) {
078                    LOG.debug("Polling directory " + fileOrDirectory);
079                    File[] files = fileOrDirectory.listFiles();
080                    for (File file : files) {
081                        rc += pollFileOrDirectory(file, isRecursive()); // self-recursion
082                    }
083                }
084                return rc;
085            } else {
086                LOG.debug("Skipping directory " + fileOrDirectory);
087                return 0;
088            }
089        }
090    
091        /**
092         * Polls the given file
093         *
094         * @param file  the file
095         * @return returns 1 if the file was processed, 0 otherwise.
096         */
097        protected int pollFile(final File file) {
098    
099            if (!file.exists()) {
100                return 0;
101            }
102            if (!isValidFile(file)) {
103                return 0;
104            }
105            // we only care about file modified times if we are not deleting/moving
106            // files
107            if (endpoint.isNoop()) {
108                long fileModified = file.lastModified();
109                if (fileModified <= lastPollTime) {
110                    if (LOG.isDebugEnabled()) {
111                        LOG.debug("Ignoring file: " + file + " as modified time: " + fileModified
112                                  + " less than last poll time: " + lastPollTime);
113                    }
114                    return 0;
115                }
116            } else {
117                if (filesBeingProcessed.contains(file)) {
118                    return 1;
119                }
120                filesBeingProcessed.put(file, file);
121            }
122    
123            final FileProcessStrategy processStrategy = endpoint.getFileStrategy();
124            final FileExchange exchange = endpoint.createExchange(file);
125    
126            endpoint.configureMessage(file, exchange.getIn());
127            try {
128                if (LOG.isDebugEnabled()) {
129                    LOG.debug("About to process file:  " + file + " using exchange: " + exchange);
130                }
131                if (processStrategy.begin(endpoint, exchange, file)) {
132    
133                    // Use the async processor interface so that processing of
134                    // the exchange can happen asynchronously
135                    getAsyncProcessor().process(exchange, new AsyncCallback() {
136                        public void done(boolean sync) {
137                            if (exchange.getException() == null) {
138                                try {
139                                    processStrategy.commit(endpoint, exchange, file);
140                                } catch (Exception e) {
141                                    handleException(e);
142                                }
143                            } else {
144                                handleException(exchange.getException());
145                            }
146                            filesBeingProcessed.remove(file);
147                        }
148                    });
149    
150                } else {
151                    if (LOG.isDebugEnabled()) {
152                        LOG.debug(endpoint + " cannot process file: " + file);
153                    }
154                }
155            } catch (Throwable e) {
156                handleException(e);
157            }
158            return 1;
159        }
160    
161        protected boolean isValidFile(File file) {
162            boolean result = false;
163            if (file != null && file.exists()) {
164                // TODO: maybe use a configurable strategy instead of the hardcoded one based on last file change
165                if (isMatched(file) && isUnchanged(file)) {
166                    result = true;
167                }
168            }
169            return result;
170        }
171    
172    
173        protected boolean isUnchanged(File file) {
174            if (file == null) {
175                // Sanity check
176                return false;
177            } else if (file.isDirectory()) {
178                // Allow recursive polling to descend into this directory
179                return true;
180            } else {
181                boolean lastModifiedCheck = true;
182                long modifiedDuration = 0;
183                if (getUnchangedDelay() > 0) {
184                    modifiedDuration = System.currentTimeMillis() - file.lastModified();
185                    lastModifiedCheck = modifiedDuration >= getUnchangedDelay();
186                }
187    
188                boolean sizeCheck = true;
189                long sizeDifference = 0;
190                if (isUnchangedSize()) {
191                    long prevFileSize = (fileSizes.get(file) == null) ? 0 : fileSizes.get(file).longValue();
192                    sizeDifference = file.length() - prevFileSize;
193                    sizeCheck = sizeDifference == 0;
194                }
195    
196                boolean answer = lastModifiedCheck && sizeCheck;
197    
198                if (LOG.isDebugEnabled()) {
199                    LOG.debug("file:" + file + " isUnchanged:" + answer + " " + "sizeCheck:" + sizeCheck + "("
200                              + sizeDifference + ") " + "lastModifiedCheck:" + lastModifiedCheck + "("
201                              + modifiedDuration + ")");
202                }
203    
204                if (isUnchangedSize()) {
205                    if (answer) {
206                        fileSizes.remove(file);
207                    } else {
208                        fileSizes.put(file, file.length());
209                    }
210                }
211    
212                return answer;
213            }
214        }
215    
216        protected boolean isMatched(File file) {
217            String name = file.getName();
218            if (regexPattern != null && regexPattern.length() > 0) {
219                if (!name.matches(getRegexPattern())) {
220                    return false;
221                }
222            }
223            String[] prefixes = endpoint.getExcludedNamePrefixes();
224            if (prefixes != null) {
225                for (String prefix : prefixes) {
226                    if (name.startsWith(prefix)) {
227                        return false;
228                    }
229                }
230            }
231            String[] postfixes = endpoint.getExcludedNamePostfixes();
232            if (postfixes != null) {
233                for (String postfix : postfixes) {
234                    if (name.endsWith(postfix)) {
235                        return false;
236                    }
237                }
238            }
239            return true;
240        }
241    
242        public boolean isRecursive() {
243            return this.recursive;
244        }
245    
246        public void setRecursive(boolean recursive) {
247            this.recursive = recursive;
248        }
249    
250        public String getRegexPattern() {
251            return this.regexPattern;
252        }
253    
254        public void setRegexPattern(String regexPattern) {
255            this.regexPattern = regexPattern;
256        }
257    
258        public boolean isGenerateEmptyExchangeWhenIdle() {
259            return generateEmptyExchangeWhenIdle;
260        }
261    
262        public void setGenerateEmptyExchangeWhenIdle(boolean generateEmptyExchangeWhenIdle) {
263            this.generateEmptyExchangeWhenIdle = generateEmptyExchangeWhenIdle;
264        }
265    
266        public int getUnchangedDelay() {
267            return unchangedDelay;
268        }
269    
270        public void setUnchangedDelay(int unchangedDelay) {
271            this.unchangedDelay = unchangedDelay;
272        }
273    
274        public boolean isUnchangedSize() {
275            return unchangedSize;
276        }
277    
278        public void setUnchangedSize(boolean unchangedSize) {
279            this.unchangedSize = unchangedSize;
280        }
281    
282    }