001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.component.file;
018
019 import java.io.File;
020 import java.util.concurrent.ConcurrentHashMap;
021
022 import org.apache.camel.AsyncCallback;
023 import org.apache.camel.Processor;
024 import org.apache.camel.impl.ScheduledPollConsumer;
025 import org.apache.commons.logging.Log;
026 import org.apache.commons.logging.LogFactory;
027
028 /**
029 * For consuming files.
030 *
031 * @version $Revision: 37863 $
032 */
033 public class FileConsumer extends ScheduledPollConsumer<FileExchange> {
034 private static final transient Log LOG = LogFactory.getLog(FileConsumer.class);
035
036 private FileEndpoint endpoint;
037 private ConcurrentHashMap<File, File> filesBeingProcessed = new ConcurrentHashMap<File, File>();
038 private ConcurrentHashMap<File, Long> fileSizes = new ConcurrentHashMap<File, Long>();
039
040 private boolean generateEmptyExchangeWhenIdle;
041 private boolean recursive = true;
042 private String regexPattern = "";
043
044 private long lastPollTime;
045 private int unchangedDelay;
046 private boolean unchangedSize;
047
048 public FileConsumer(final FileEndpoint endpoint, Processor processor) {
049 super(endpoint, processor);
050 this.endpoint = endpoint;
051 }
052
053 protected synchronized void poll() throws Exception {
054 int rc = pollFileOrDirectory(endpoint.getFile(), isRecursive());
055 if (rc == 0 && generateEmptyExchangeWhenIdle) {
056 final FileExchange exchange = endpoint.createExchange((File)null);
057 getAsyncProcessor().process(exchange, new AsyncCallback() {
058 public void done(boolean sync) {
059 }
060 });
061 }
062 lastPollTime = System.currentTimeMillis();
063 }
064
065 /**
066 * Pools the given file or directory for files to process.
067 *
068 * @param fileOrDirectory file or directory
069 * @param processDir recursive
070 * @return the number of files processed or being processed async.
071 */
072 protected int pollFileOrDirectory(File fileOrDirectory, boolean processDir) {
073 if (!fileOrDirectory.isDirectory()) {
074 return pollFile(fileOrDirectory); // process the file
075 } else if (processDir) {
076 int rc = 0;
077 if (isValidFile(fileOrDirectory)) {
078 LOG.debug("Polling directory " + fileOrDirectory);
079 File[] files = fileOrDirectory.listFiles();
080 for (File file : files) {
081 rc += pollFileOrDirectory(file, isRecursive()); // self-recursion
082 }
083 }
084 return rc;
085 } else {
086 LOG.debug("Skipping directory " + fileOrDirectory);
087 return 0;
088 }
089 }
090
091 /**
092 * Polls the given file
093 *
094 * @param file the file
095 * @return returns 1 if the file was processed, 0 otherwise.
096 */
097 protected int pollFile(final File file) {
098
099 if (!file.exists()) {
100 return 0;
101 }
102 if (!isValidFile(file)) {
103 return 0;
104 }
105 // we only care about file modified times if we are not deleting/moving
106 // files
107 if (endpoint.isNoop()) {
108 long fileModified = file.lastModified();
109 if (fileModified <= lastPollTime) {
110 if (LOG.isDebugEnabled()) {
111 LOG.debug("Ignoring file: " + file + " as modified time: " + fileModified
112 + " less than last poll time: " + lastPollTime);
113 }
114 return 0;
115 }
116 } else {
117 if (filesBeingProcessed.contains(file)) {
118 return 1;
119 }
120 filesBeingProcessed.put(file, file);
121 }
122
123 final FileProcessStrategy processStrategy = endpoint.getFileStrategy();
124 final FileExchange exchange = endpoint.createExchange(file);
125
126 endpoint.configureMessage(file, exchange.getIn());
127 try {
128 if (LOG.isDebugEnabled()) {
129 LOG.debug("About to process file: " + file + " using exchange: " + exchange);
130 }
131 if (processStrategy.begin(endpoint, exchange, file)) {
132
133 // Use the async processor interface so that processing of
134 // the exchange can happen asynchronously
135 getAsyncProcessor().process(exchange, new AsyncCallback() {
136 public void done(boolean sync) {
137 if (exchange.getException() == null) {
138 try {
139 processStrategy.commit(endpoint, exchange, file);
140 } catch (Exception e) {
141 handleException(e);
142 }
143 } else {
144 handleException(exchange.getException());
145 }
146 filesBeingProcessed.remove(file);
147 }
148 });
149
150 } else {
151 if (LOG.isDebugEnabled()) {
152 LOG.debug(endpoint + " cannot process file: " + file);
153 }
154 }
155 } catch (Throwable e) {
156 handleException(e);
157 }
158 return 1;
159 }
160
161 protected boolean isValidFile(File file) {
162 boolean result = false;
163 if (file != null && file.exists()) {
164 // TODO: maybe use a configurable strategy instead of the hardcoded one based on last file change
165 if (isMatched(file) && isUnchanged(file)) {
166 result = true;
167 }
168 }
169 return result;
170 }
171
172
173 protected boolean isUnchanged(File file) {
174 if (file == null) {
175 // Sanity check
176 return false;
177 } else if (file.isDirectory()) {
178 // Allow recursive polling to descend into this directory
179 return true;
180 } else {
181 boolean lastModifiedCheck = true;
182 long modifiedDuration = 0;
183 if (getUnchangedDelay() > 0) {
184 modifiedDuration = System.currentTimeMillis() - file.lastModified();
185 lastModifiedCheck = modifiedDuration >= getUnchangedDelay();
186 }
187
188 boolean sizeCheck = true;
189 long sizeDifference = 0;
190 if (isUnchangedSize()) {
191 long prevFileSize = (fileSizes.get(file) == null) ? 0 : fileSizes.get(file).longValue();
192 sizeDifference = file.length() - prevFileSize;
193 sizeCheck = sizeDifference == 0;
194 }
195
196 boolean answer = lastModifiedCheck && sizeCheck;
197
198 if (LOG.isDebugEnabled()) {
199 LOG.debug("file:" + file + " isUnchanged:" + answer + " " + "sizeCheck:" + sizeCheck + "("
200 + sizeDifference + ") " + "lastModifiedCheck:" + lastModifiedCheck + "("
201 + modifiedDuration + ")");
202 }
203
204 if (isUnchangedSize()) {
205 if (answer) {
206 fileSizes.remove(file);
207 } else {
208 fileSizes.put(file, file.length());
209 }
210 }
211
212 return answer;
213 }
214 }
215
216 protected boolean isMatched(File file) {
217 String name = file.getName();
218 if (regexPattern != null && regexPattern.length() > 0) {
219 if (!name.matches(getRegexPattern())) {
220 return false;
221 }
222 }
223 String[] prefixes = endpoint.getExcludedNamePrefixes();
224 if (prefixes != null) {
225 for (String prefix : prefixes) {
226 if (name.startsWith(prefix)) {
227 return false;
228 }
229 }
230 }
231 String[] postfixes = endpoint.getExcludedNamePostfixes();
232 if (postfixes != null) {
233 for (String postfix : postfixes) {
234 if (name.endsWith(postfix)) {
235 return false;
236 }
237 }
238 }
239 return true;
240 }
241
242 public boolean isRecursive() {
243 return this.recursive;
244 }
245
246 public void setRecursive(boolean recursive) {
247 this.recursive = recursive;
248 }
249
250 public String getRegexPattern() {
251 return this.regexPattern;
252 }
253
254 public void setRegexPattern(String regexPattern) {
255 this.regexPattern = regexPattern;
256 }
257
258 public boolean isGenerateEmptyExchangeWhenIdle() {
259 return generateEmptyExchangeWhenIdle;
260 }
261
262 public void setGenerateEmptyExchangeWhenIdle(boolean generateEmptyExchangeWhenIdle) {
263 this.generateEmptyExchangeWhenIdle = generateEmptyExchangeWhenIdle;
264 }
265
266 public int getUnchangedDelay() {
267 return unchangedDelay;
268 }
269
270 public void setUnchangedDelay(int unchangedDelay) {
271 this.unchangedDelay = unchangedDelay;
272 }
273
274 public boolean isUnchangedSize() {
275 return unchangedSize;
276 }
277
278 public void setUnchangedSize(boolean unchangedSize) {
279 this.unchangedSize = unchangedSize;
280 }
281
282 }