001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 package org.apache.camel.component.file; 018 019 import java.io.File; 020 import java.util.concurrent.ConcurrentHashMap; 021 022 import org.apache.camel.AsyncCallback; 023 import org.apache.camel.Processor; 024 import org.apache.camel.impl.ScheduledPollConsumer; 025 import org.apache.commons.logging.Log; 026 import org.apache.commons.logging.LogFactory; 027 028 /** 029 * For consuming files. 030 * 031 * @version $Revision: 37863 $ 032 */ 033 public class FileConsumer extends ScheduledPollConsumer<FileExchange> { 034 private static final transient Log LOG = LogFactory.getLog(FileConsumer.class); 035 036 private FileEndpoint endpoint; 037 private ConcurrentHashMap<File, File> filesBeingProcessed = new ConcurrentHashMap<File, File>(); 038 private ConcurrentHashMap<File, Long> fileSizes = new ConcurrentHashMap<File, Long>(); 039 040 private boolean generateEmptyExchangeWhenIdle; 041 private boolean recursive = true; 042 private String regexPattern = ""; 043 044 private long lastPollTime; 045 private int unchangedDelay; 046 private boolean unchangedSize; 047 048 public FileConsumer(final FileEndpoint endpoint, Processor processor) { 049 super(endpoint, processor); 050 this.endpoint = endpoint; 051 } 052 053 protected synchronized void poll() throws Exception { 054 int rc = pollFileOrDirectory(endpoint.getFile(), isRecursive()); 055 if (rc == 0 && generateEmptyExchangeWhenIdle) { 056 final FileExchange exchange = endpoint.createExchange((File)null); 057 getAsyncProcessor().process(exchange, new AsyncCallback() { 058 public void done(boolean sync) { 059 } 060 }); 061 } 062 lastPollTime = System.currentTimeMillis(); 063 } 064 065 /** 066 * Pools the given file or directory for files to process. 067 * 068 * @param fileOrDirectory file or directory 069 * @param processDir recursive 070 * @return the number of files processed or being processed async. 071 */ 072 protected int pollFileOrDirectory(File fileOrDirectory, boolean processDir) { 073 if (!fileOrDirectory.isDirectory()) { 074 return pollFile(fileOrDirectory); // process the file 075 } else if (processDir) { 076 int rc = 0; 077 if (isValidFile(fileOrDirectory)) { 078 LOG.debug("Polling directory " + fileOrDirectory); 079 File[] files = fileOrDirectory.listFiles(); 080 for (File file : files) { 081 rc += pollFileOrDirectory(file, isRecursive()); // self-recursion 082 } 083 } 084 return rc; 085 } else { 086 LOG.debug("Skipping directory " + fileOrDirectory); 087 return 0; 088 } 089 } 090 091 /** 092 * Polls the given file 093 * 094 * @param file the file 095 * @return returns 1 if the file was processed, 0 otherwise. 096 */ 097 protected int pollFile(final File file) { 098 099 if (!file.exists()) { 100 return 0; 101 } 102 if (!isValidFile(file)) { 103 return 0; 104 } 105 // we only care about file modified times if we are not deleting/moving 106 // files 107 if (endpoint.isNoop()) { 108 long fileModified = file.lastModified(); 109 if (fileModified <= lastPollTime) { 110 if (LOG.isDebugEnabled()) { 111 LOG.debug("Ignoring file: " + file + " as modified time: " + fileModified 112 + " less than last poll time: " + lastPollTime); 113 } 114 return 0; 115 } 116 } else { 117 if (filesBeingProcessed.contains(file)) { 118 return 1; 119 } 120 filesBeingProcessed.put(file, file); 121 } 122 123 final FileProcessStrategy processStrategy = endpoint.getFileStrategy(); 124 final FileExchange exchange = endpoint.createExchange(file); 125 126 endpoint.configureMessage(file, exchange.getIn()); 127 try { 128 if (LOG.isDebugEnabled()) { 129 LOG.debug("About to process file: " + file + " using exchange: " + exchange); 130 } 131 if (processStrategy.begin(endpoint, exchange, file)) { 132 133 // Use the async processor interface so that processing of 134 // the exchange can happen asynchronously 135 getAsyncProcessor().process(exchange, new AsyncCallback() { 136 public void done(boolean sync) { 137 if (exchange.getException() == null) { 138 try { 139 processStrategy.commit(endpoint, exchange, file); 140 } catch (Exception e) { 141 handleException(e); 142 } 143 } else { 144 handleException(exchange.getException()); 145 } 146 filesBeingProcessed.remove(file); 147 } 148 }); 149 150 } else { 151 if (LOG.isDebugEnabled()) { 152 LOG.debug(endpoint + " cannot process file: " + file); 153 } 154 } 155 } catch (Throwable e) { 156 handleException(e); 157 } 158 return 1; 159 } 160 161 protected boolean isValidFile(File file) { 162 boolean result = false; 163 if (file != null && file.exists()) { 164 // TODO: maybe use a configurable strategy instead of the hardcoded one based on last file change 165 if (isMatched(file) && isUnchanged(file)) { 166 result = true; 167 } 168 } 169 return result; 170 } 171 172 173 protected boolean isUnchanged(File file) { 174 if (file == null) { 175 // Sanity check 176 return false; 177 } else if (file.isDirectory()) { 178 // Allow recursive polling to descend into this directory 179 return true; 180 } else { 181 boolean lastModifiedCheck = true; 182 long modifiedDuration = 0; 183 if (getUnchangedDelay() > 0) { 184 modifiedDuration = System.currentTimeMillis() - file.lastModified(); 185 lastModifiedCheck = modifiedDuration >= getUnchangedDelay(); 186 } 187 188 boolean sizeCheck = true; 189 long sizeDifference = 0; 190 if (isUnchangedSize()) { 191 long prevFileSize = (fileSizes.get(file) == null) ? 0 : fileSizes.get(file).longValue(); 192 sizeDifference = file.length() - prevFileSize; 193 sizeCheck = sizeDifference == 0; 194 } 195 196 boolean answer = lastModifiedCheck && sizeCheck; 197 198 if (LOG.isDebugEnabled()) { 199 LOG.debug("file:" + file + " isUnchanged:" + answer + " " + "sizeCheck:" + sizeCheck + "(" 200 + sizeDifference + ") " + "lastModifiedCheck:" + lastModifiedCheck + "(" 201 + modifiedDuration + ")"); 202 } 203 204 if (isUnchangedSize()) { 205 if (answer) { 206 fileSizes.remove(file); 207 } else { 208 fileSizes.put(file, file.length()); 209 } 210 } 211 212 return answer; 213 } 214 } 215 216 protected boolean isMatched(File file) { 217 String name = file.getName(); 218 if (regexPattern != null && regexPattern.length() > 0) { 219 if (!name.matches(getRegexPattern())) { 220 return false; 221 } 222 } 223 String[] prefixes = endpoint.getExcludedNamePrefixes(); 224 if (prefixes != null) { 225 for (String prefix : prefixes) { 226 if (name.startsWith(prefix)) { 227 return false; 228 } 229 } 230 } 231 String[] postfixes = endpoint.getExcludedNamePostfixes(); 232 if (postfixes != null) { 233 for (String postfix : postfixes) { 234 if (name.endsWith(postfix)) { 235 return false; 236 } 237 } 238 } 239 return true; 240 } 241 242 public boolean isRecursive() { 243 return this.recursive; 244 } 245 246 public void setRecursive(boolean recursive) { 247 this.recursive = recursive; 248 } 249 250 public String getRegexPattern() { 251 return this.regexPattern; 252 } 253 254 public void setRegexPattern(String regexPattern) { 255 this.regexPattern = regexPattern; 256 } 257 258 public boolean isGenerateEmptyExchangeWhenIdle() { 259 return generateEmptyExchangeWhenIdle; 260 } 261 262 public void setGenerateEmptyExchangeWhenIdle(boolean generateEmptyExchangeWhenIdle) { 263 this.generateEmptyExchangeWhenIdle = generateEmptyExchangeWhenIdle; 264 } 265 266 public int getUnchangedDelay() { 267 return unchangedDelay; 268 } 269 270 public void setUnchangedDelay(int unchangedDelay) { 271 this.unchangedDelay = unchangedDelay; 272 } 273 274 public boolean isUnchangedSize() { 275 return unchangedSize; 276 } 277 278 public void setUnchangedSize(boolean unchangedSize) { 279 this.unchangedSize = unchangedSize; 280 } 281 282 }