001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.component.file;
018
019 import java.io.File;
020 import java.util.concurrent.ConcurrentHashMap;
021
022 import org.apache.camel.AsyncCallback;
023 import org.apache.camel.Processor;
024 import org.apache.camel.impl.ScheduledPollConsumer;
025 import org.apache.commons.logging.Log;
026 import org.apache.commons.logging.LogFactory;
027
028 /**
029 * For consuming files.
030 *
031 * @version $Revision: 41272 $
032 */
033 public class FileConsumer extends ScheduledPollConsumer<FileExchange> {
034 private static final transient Log LOG = LogFactory.getLog(FileConsumer.class);
035
036 private FileEndpoint endpoint;
037 private ConcurrentHashMap<File, File> filesBeingProcessed = new ConcurrentHashMap<File, File>();
038 private ConcurrentHashMap<File, Long> fileSizes = new ConcurrentHashMap<File, Long>();
039 private ConcurrentHashMap<File, Long> noopMap = new ConcurrentHashMap<File, Long>();
040
041 private boolean generateEmptyExchangeWhenIdle;
042 private boolean recursive = true;
043 private String regexPattern = "";
044
045 private long lastPollTime;
046 private int unchangedDelay;
047 private boolean unchangedSize;
048
049
050 public FileConsumer(final FileEndpoint endpoint, Processor processor) {
051 super(endpoint, processor);
052 this.endpoint = endpoint;
053 }
054
055 protected synchronized void poll() throws Exception {
056 int rc = pollFileOrDirectory(endpoint.getFile(), isRecursive());
057 if (rc == 0 && generateEmptyExchangeWhenIdle) {
058 final FileExchange exchange = endpoint.createExchange((File)null);
059 getAsyncProcessor().process(exchange, new AsyncCallback() {
060 public void done(boolean sync) {
061 }
062 });
063 }
064 lastPollTime = System.currentTimeMillis();
065 }
066
067 /**
068 * Pools the given file or directory for files to process.
069 *
070 * @param fileOrDirectory file or directory
071 * @param processDir recursive
072 * @return the number of files processed or being processed async.
073 */
074 protected int pollFileOrDirectory(File fileOrDirectory, boolean processDir) {
075 if (!fileOrDirectory.isDirectory()) {
076 return pollFile(fileOrDirectory); // process the file
077 } else if (processDir) {
078 int rc = 0;
079 if (isValidFile(fileOrDirectory)) {
080 LOG.debug("Polling directory " + fileOrDirectory);
081 File[] files = fileOrDirectory.listFiles();
082 for (File file : files) {
083 rc += pollFileOrDirectory(file, isRecursive()); // self-recursion
084 }
085 }
086 return rc;
087 } else {
088 LOG.debug("Skipping directory " + fileOrDirectory);
089 return 0;
090 }
091 }
092
093 /**
094 * Polls the given file
095 *
096 * @param file the file
097 * @return returns 1 if the file was processed, 0 otherwise.
098 */
099 protected int pollFile(final File file) {
100
101 if (!file.exists()) {
102 return 0;
103 }
104 if (!isValidFile(file)) {
105 return 0;
106 }
107 // we only care about file modified times if we are not deleting/moving
108 // files
109 if (!endpoint.isNoop()) {
110 if (filesBeingProcessed.contains(file)) {
111 return 1;
112 }
113 filesBeingProcessed.put(file, file);
114 }
115
116 final FileProcessStrategy processStrategy = endpoint.getFileStrategy();
117 final FileExchange exchange = endpoint.createExchange(file);
118
119 endpoint.configureMessage(file, exchange.getIn());
120 try {
121 if (LOG.isDebugEnabled()) {
122 LOG.debug("About to process file: " + file + " using exchange: " + exchange);
123 }
124 if (processStrategy.begin(endpoint, exchange, file)) {
125
126 // Use the async processor interface so that processing of
127 // the exchange can happen asynchronously
128 getAsyncProcessor().process(exchange, new AsyncCallback() {
129 public void done(boolean sync) {
130 if (exchange.getException() == null) {
131 try {
132 processStrategy.commit(endpoint, exchange, file);
133 } catch (Exception e) {
134 handleException(e);
135 }
136 } else {
137 handleException(exchange.getException());
138 }
139 filesBeingProcessed.remove(file);
140 }
141 });
142
143 } else {
144 if (LOG.isDebugEnabled()) {
145 LOG.debug(endpoint + " cannot process file: " + file);
146 }
147 }
148 } catch (Throwable e) {
149 handleException(e);
150 }
151 return 1;
152 }
153
154 protected boolean isValidFile(File file) {
155 boolean result = false;
156 if (file != null && file.exists()) {
157 // TODO: maybe use a configurable strategy instead of the hardcoded one based on last file change
158 if (isMatched(file) && isChanged(file)) {
159 result = true;
160 }
161 }
162 return result;
163 }
164
165
166 protected boolean isChanged(File file) {
167 if (file == null) {
168 // Sanity check
169 return false;
170 } else if (file.isDirectory()) {
171 // Allow recursive polling to descend into this directory
172 return true;
173 } else {
174 boolean lastModifiedCheck = false;
175 long modifiedDuration = 0;
176 if (getUnchangedDelay() > 0) {
177 modifiedDuration = System.currentTimeMillis() - file.lastModified();
178 lastModifiedCheck = modifiedDuration >= getUnchangedDelay();
179 }
180
181 long fileModified = file.lastModified();
182 Long previousModified = noopMap.get(file);
183 noopMap.put(file, fileModified);
184 if (previousModified == null || fileModified > previousModified) {
185 lastModifiedCheck = true;
186 }
187
188 boolean sizeCheck = false;
189 long sizeDifference = 0;
190 if (isUnchangedSize()) {
191 Long value = fileSizes.get(file);
192 if (value == null) {
193 sizeCheck = true;
194 } else {
195 sizeCheck = file.length() != value;
196 }
197 }
198
199 boolean answer = lastModifiedCheck || sizeCheck;
200
201 if (LOG.isDebugEnabled()) {
202 LOG.debug("file:" + file + " isChanged:" + answer + " " + "sizeCheck:" + sizeCheck + "("
203 + sizeDifference + ") " + "lastModifiedCheck:" + lastModifiedCheck + "("
204 + modifiedDuration + ")");
205 }
206
207 if (isUnchangedSize()) {
208 if (answer) {
209 fileSizes.put(file, file.length());
210 } else {
211 fileSizes.remove(file);
212 }
213 }
214
215 return answer;
216 }
217 }
218
219 protected boolean isMatched(File file) {
220 String name = file.getName();
221 if (regexPattern != null && regexPattern.length() > 0) {
222 if (!name.matches(getRegexPattern())) {
223 return false;
224 }
225 }
226 String[] prefixes = endpoint.getExcludedNamePrefixes();
227 if (prefixes != null) {
228 for (String prefix : prefixes) {
229 if (name.startsWith(prefix)) {
230 return false;
231 }
232 }
233 }
234 String[] postfixes = endpoint.getExcludedNamePostfixes();
235 if (postfixes != null) {
236 for (String postfix : postfixes) {
237 if (name.endsWith(postfix)) {
238 return false;
239 }
240 }
241 }
242 return true;
243 }
244
245 public boolean isRecursive() {
246 return this.recursive;
247 }
248
249 public void setRecursive(boolean recursive) {
250 this.recursive = recursive;
251 }
252
253 public String getRegexPattern() {
254 return this.regexPattern;
255 }
256
257 public void setRegexPattern(String regexPattern) {
258 this.regexPattern = regexPattern;
259 }
260
261 public boolean isGenerateEmptyExchangeWhenIdle() {
262 return generateEmptyExchangeWhenIdle;
263 }
264
265 public void setGenerateEmptyExchangeWhenIdle(boolean generateEmptyExchangeWhenIdle) {
266 this.generateEmptyExchangeWhenIdle = generateEmptyExchangeWhenIdle;
267 }
268
269 public int getUnchangedDelay() {
270 return unchangedDelay;
271 }
272
273 public void setUnchangedDelay(int unchangedDelay) {
274 this.unchangedDelay = unchangedDelay;
275 }
276
277 public boolean isUnchangedSize() {
278 return unchangedSize;
279 }
280
281 public void setUnchangedSize(boolean unchangedSize) {
282 this.unchangedSize = unchangedSize;
283 }
284
285 }