001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 package org.apache.camel.component.file; 018 019 import java.io.File; 020 import java.util.concurrent.ConcurrentHashMap; 021 022 import org.apache.camel.AsyncCallback; 023 import org.apache.camel.Processor; 024 import org.apache.camel.impl.ScheduledPollConsumer; 025 import org.apache.commons.logging.Log; 026 import org.apache.commons.logging.LogFactory; 027 028 /** 029 * For consuming files. 030 * 031 * @version $Revision: 41272 $ 032 */ 033 public class FileConsumer extends ScheduledPollConsumer<FileExchange> { 034 private static final transient Log LOG = LogFactory.getLog(FileConsumer.class); 035 036 private FileEndpoint endpoint; 037 private ConcurrentHashMap<File, File> filesBeingProcessed = new ConcurrentHashMap<File, File>(); 038 private ConcurrentHashMap<File, Long> fileSizes = new ConcurrentHashMap<File, Long>(); 039 private ConcurrentHashMap<File, Long> noopMap = new ConcurrentHashMap<File, Long>(); 040 041 private boolean generateEmptyExchangeWhenIdle; 042 private boolean recursive = true; 043 private String regexPattern = ""; 044 045 private long lastPollTime; 046 private int unchangedDelay; 047 private boolean unchangedSize; 048 049 050 public FileConsumer(final FileEndpoint endpoint, Processor processor) { 051 super(endpoint, processor); 052 this.endpoint = endpoint; 053 } 054 055 protected synchronized void poll() throws Exception { 056 int rc = pollFileOrDirectory(endpoint.getFile(), isRecursive()); 057 if (rc == 0 && generateEmptyExchangeWhenIdle) { 058 final FileExchange exchange = endpoint.createExchange((File)null); 059 getAsyncProcessor().process(exchange, new AsyncCallback() { 060 public void done(boolean sync) { 061 } 062 }); 063 } 064 lastPollTime = System.currentTimeMillis(); 065 } 066 067 /** 068 * Pools the given file or directory for files to process. 069 * 070 * @param fileOrDirectory file or directory 071 * @param processDir recursive 072 * @return the number of files processed or being processed async. 073 */ 074 protected int pollFileOrDirectory(File fileOrDirectory, boolean processDir) { 075 if (!fileOrDirectory.isDirectory()) { 076 return pollFile(fileOrDirectory); // process the file 077 } else if (processDir) { 078 int rc = 0; 079 if (isValidFile(fileOrDirectory)) { 080 LOG.debug("Polling directory " + fileOrDirectory); 081 File[] files = fileOrDirectory.listFiles(); 082 for (File file : files) { 083 rc += pollFileOrDirectory(file, isRecursive()); // self-recursion 084 } 085 } 086 return rc; 087 } else { 088 LOG.debug("Skipping directory " + fileOrDirectory); 089 return 0; 090 } 091 } 092 093 /** 094 * Polls the given file 095 * 096 * @param file the file 097 * @return returns 1 if the file was processed, 0 otherwise. 098 */ 099 protected int pollFile(final File file) { 100 101 if (!file.exists()) { 102 return 0; 103 } 104 if (!isValidFile(file)) { 105 return 0; 106 } 107 // we only care about file modified times if we are not deleting/moving 108 // files 109 if (!endpoint.isNoop()) { 110 if (filesBeingProcessed.contains(file)) { 111 return 1; 112 } 113 filesBeingProcessed.put(file, file); 114 } 115 116 final FileProcessStrategy processStrategy = endpoint.getFileStrategy(); 117 final FileExchange exchange = endpoint.createExchange(file); 118 119 endpoint.configureMessage(file, exchange.getIn()); 120 try { 121 if (LOG.isDebugEnabled()) { 122 LOG.debug("About to process file: " + file + " using exchange: " + exchange); 123 } 124 if (processStrategy.begin(endpoint, exchange, file)) { 125 126 // Use the async processor interface so that processing of 127 // the exchange can happen asynchronously 128 getAsyncProcessor().process(exchange, new AsyncCallback() { 129 public void done(boolean sync) { 130 if (exchange.getException() == null) { 131 try { 132 processStrategy.commit(endpoint, exchange, file); 133 } catch (Exception e) { 134 handleException(e); 135 } 136 } else { 137 handleException(exchange.getException()); 138 } 139 filesBeingProcessed.remove(file); 140 } 141 }); 142 143 } else { 144 if (LOG.isDebugEnabled()) { 145 LOG.debug(endpoint + " cannot process file: " + file); 146 } 147 } 148 } catch (Throwable e) { 149 handleException(e); 150 } 151 return 1; 152 } 153 154 protected boolean isValidFile(File file) { 155 boolean result = false; 156 if (file != null && file.exists()) { 157 // TODO: maybe use a configurable strategy instead of the hardcoded one based on last file change 158 if (isMatched(file) && isChanged(file)) { 159 result = true; 160 } 161 } 162 return result; 163 } 164 165 166 protected boolean isChanged(File file) { 167 if (file == null) { 168 // Sanity check 169 return false; 170 } else if (file.isDirectory()) { 171 // Allow recursive polling to descend into this directory 172 return true; 173 } else { 174 boolean lastModifiedCheck = false; 175 long modifiedDuration = 0; 176 if (getUnchangedDelay() > 0) { 177 modifiedDuration = System.currentTimeMillis() - file.lastModified(); 178 lastModifiedCheck = modifiedDuration >= getUnchangedDelay(); 179 } 180 181 long fileModified = file.lastModified(); 182 Long previousModified = noopMap.get(file); 183 noopMap.put(file, fileModified); 184 if (previousModified == null || fileModified > previousModified) { 185 lastModifiedCheck = true; 186 } 187 188 boolean sizeCheck = false; 189 long sizeDifference = 0; 190 if (isUnchangedSize()) { 191 Long value = fileSizes.get(file); 192 if (value == null) { 193 sizeCheck = true; 194 } else { 195 sizeCheck = file.length() != value; 196 } 197 } 198 199 boolean answer = lastModifiedCheck || sizeCheck; 200 201 if (LOG.isDebugEnabled()) { 202 LOG.debug("file:" + file + " isChanged:" + answer + " " + "sizeCheck:" + sizeCheck + "(" 203 + sizeDifference + ") " + "lastModifiedCheck:" + lastModifiedCheck + "(" 204 + modifiedDuration + ")"); 205 } 206 207 if (isUnchangedSize()) { 208 if (answer) { 209 fileSizes.put(file, file.length()); 210 } else { 211 fileSizes.remove(file); 212 } 213 } 214 215 return answer; 216 } 217 } 218 219 protected boolean isMatched(File file) { 220 String name = file.getName(); 221 if (regexPattern != null && regexPattern.length() > 0) { 222 if (!name.matches(getRegexPattern())) { 223 return false; 224 } 225 } 226 String[] prefixes = endpoint.getExcludedNamePrefixes(); 227 if (prefixes != null) { 228 for (String prefix : prefixes) { 229 if (name.startsWith(prefix)) { 230 return false; 231 } 232 } 233 } 234 String[] postfixes = endpoint.getExcludedNamePostfixes(); 235 if (postfixes != null) { 236 for (String postfix : postfixes) { 237 if (name.endsWith(postfix)) { 238 return false; 239 } 240 } 241 } 242 return true; 243 } 244 245 public boolean isRecursive() { 246 return this.recursive; 247 } 248 249 public void setRecursive(boolean recursive) { 250 this.recursive = recursive; 251 } 252 253 public String getRegexPattern() { 254 return this.regexPattern; 255 } 256 257 public void setRegexPattern(String regexPattern) { 258 this.regexPattern = regexPattern; 259 } 260 261 public boolean isGenerateEmptyExchangeWhenIdle() { 262 return generateEmptyExchangeWhenIdle; 263 } 264 265 public void setGenerateEmptyExchangeWhenIdle(boolean generateEmptyExchangeWhenIdle) { 266 this.generateEmptyExchangeWhenIdle = generateEmptyExchangeWhenIdle; 267 } 268 269 public int getUnchangedDelay() { 270 return unchangedDelay; 271 } 272 273 public void setUnchangedDelay(int unchangedDelay) { 274 this.unchangedDelay = unchangedDelay; 275 } 276 277 public boolean isUnchangedSize() { 278 return unchangedSize; 279 } 280 281 public void setUnchangedSize(boolean unchangedSize) { 282 this.unchangedSize = unchangedSize; 283 } 284 285 }