001 /*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.servicemix.ftp;
018
019 import java.io.File;
020 import java.io.FileFilter;
021 import java.io.IOException;
022 import java.io.InputStream;
023 import java.net.URI;
024 import java.util.concurrent.ConcurrentHashMap;
025 import java.util.concurrent.ConcurrentMap;
026 import java.util.concurrent.locks.Lock;
027
028 import javax.jbi.JBIException;
029 import javax.jbi.management.DeploymentException;
030 import javax.jbi.messaging.ExchangeStatus;
031 import javax.jbi.messaging.InOnly;
032 import javax.jbi.messaging.MessageExchange;
033 import javax.jbi.messaging.NormalizedMessage;
034 import javax.jbi.servicedesc.ServiceEndpoint;
035 import javax.xml.namespace.QName;
036
037 import org.apache.commons.net.ftp.FTPClient;
038 import org.apache.commons.net.ftp.FTPFile;
039 import org.apache.servicemix.common.DefaultComponent;
040 import org.apache.servicemix.common.ServiceUnit;
041 import org.apache.servicemix.common.endpoints.PollingEndpoint;
042 import org.apache.servicemix.common.locks.LockManager;
043 import org.apache.servicemix.common.locks.impl.SimpleLockManager;
044 import org.apache.servicemix.components.util.DefaultFileMarshaler;
045 import org.apache.servicemix.components.util.FileMarshaler;
046
047 /**
048 * A polling endpoint which looks for a file or files in a directory
049 * and sends the files into the JBI bus as messages, deleting the files
050 * by default when they are processed.
051 *
052 * @org.apache.xbean.XBean element="poller"
053 *
054 * @version $Revision: 468487 $
055 */
056 public class FtpPollerEndpoint extends PollingEndpoint implements FtpEndpointType {
057
058 private FTPClientPool clientPool;
059 private FileFilter filter;
060 private boolean deleteFile = true;
061 private boolean recursive = true;
062 private boolean changeWorkingDirectory;
063 private FileMarshaler marshaler = new DefaultFileMarshaler();
064 private LockManager lockManager;
065 private ConcurrentMap<String, FtpData> openExchanges = new ConcurrentHashMap<String, FtpData>();
066 private QName targetOperation;
067 private URI uri;
068 private boolean stateless = true;
069
070 protected class FtpData {
071 final String file;
072 final FTPClient ftp;
073 final InputStream in;
074 public FtpData(String file, FTPClient ftp, InputStream in) {
075 this.file = file;
076 this.ftp = ftp;
077 this.in = in;
078 }
079 }
080
081 public FtpPollerEndpoint() {
082 }
083
084 public FtpPollerEndpoint(ServiceUnit serviceUnit, QName service, String endpoint) {
085 super(serviceUnit, service, endpoint);
086 }
087
088 public FtpPollerEndpoint(DefaultComponent component, ServiceEndpoint endpoint) {
089 super(component, endpoint);
090 }
091
092 public void poll() throws Exception {
093 pollFileOrDirectory(getWorkingPath());
094 }
095
096 public void validate() throws DeploymentException {
097 super.validate();
098 if (uri == null && (getClientPool() == null || getClientPool().getHost() == null)) {
099 throw new DeploymentException("Property uri or clientPool.host must be configured");
100 }
101 if (uri != null && getClientPool() != null && getClientPool().getHost() != null) {
102 throw new DeploymentException("Properties uri and clientPool.host can not be configured at the same time");
103 }
104 if (changeWorkingDirectory && recursive) {
105 throw new DeploymentException("changeWorkingDirectory='true' can not be set when recursive='true'");
106 }
107 }
108
109 public void start() throws Exception {
110 if (lockManager == null) {
111 lockManager = createLockManager();
112 }
113 if (clientPool == null) {
114 clientPool = createClientPool();
115 }
116 if (uri != null) {
117 clientPool.setHost(uri.getHost());
118 clientPool.setPort(uri.getPort());
119 if (uri.getUserInfo() != null) {
120 String[] infos = uri.getUserInfo().split(":");
121 clientPool.setUsername(infos[0]);
122 if (infos.length > 1) {
123 clientPool.setPassword(infos[1]);
124 }
125 }
126 } else {
127 String str = "ftp://" + clientPool.getHost();
128 if (clientPool.getPort() >= 0) {
129 str += ":" + clientPool.getPort();
130 }
131 str += "/";
132 uri = new URI(str);
133 }
134 super.start();
135 }
136
137 protected LockManager createLockManager() {
138 return new SimpleLockManager();
139 }
140
141 private String getWorkingPath() {
142 return (uri != null && uri.getPath() != null) ? uri.getPath() : ".";
143 }
144
145 // Properties
146 //-------------------------------------------------------------------------
147 /**
148 * @return the clientPool
149 */
150 public FTPClientPool getClientPool() {
151 return clientPool;
152 }
153
154 /**
155 * @param clientPool the clientPool to set
156 */
157 public void setClientPool(FTPClientPool clientPool) {
158 this.clientPool = clientPool;
159 }
160
161 /**
162 * @return the uri
163 */
164 public URI getUri() {
165 return uri;
166 }
167
168 /**
169 * @param uri the uri to set
170 */
171 public void setUri(URI uri) {
172 this.uri = uri;
173 }
174
175 public FileFilter getFilter() {
176 return filter;
177 }
178
179 /**
180 * Sets the optional filter to choose which files to process
181 */
182 public void setFilter(FileFilter filter) {
183 this.filter = filter;
184 }
185
186 /**
187 * Returns whether or not we should delete the file when its processed
188 */
189 public boolean isDeleteFile() {
190 return deleteFile;
191 }
192
193 public void setDeleteFile(boolean deleteFile) {
194 this.deleteFile = deleteFile;
195 }
196
197 public boolean isRecursive() {
198 return recursive;
199 }
200
201 public void setRecursive(boolean recursive) {
202 this.recursive = recursive;
203 }
204
205 public FileMarshaler getMarshaler() {
206 return marshaler;
207 }
208
209 public void setMarshaler(FileMarshaler marshaler) {
210 this.marshaler = marshaler;
211 }
212
213 public QName getTargetOperation() {
214 return targetOperation;
215 }
216
217 public void setTargetOperation(QName targetOperation) {
218 this.targetOperation = targetOperation;
219 }
220
221 public void setChangeWorkingDirectory(boolean changeWorkingDirectory) {
222 this.changeWorkingDirectory = changeWorkingDirectory;
223 }
224
225 public boolean isStateless() {
226 return stateless;
227 }
228
229 public void setStateless(boolean stateless) {
230 this.stateless = stateless;
231 }
232
233 // Implementation methods
234 //-------------------------------------------------------------------------
235
236 protected void pollFileOrDirectory(String fileOrDirectory) throws Exception {
237 FTPClient ftp = borrowClient();
238 try {
239 logger.debug("Polling directory " + fileOrDirectory);
240 pollFileOrDirectory(ftp, fileOrDirectory, isRecursive());
241 } finally {
242 returnClient(ftp);
243 }
244 }
245
246 protected void pollFileOrDirectory(FTPClient ftp, String fileOrDirectory, boolean processDir) throws Exception {
247 FTPFile[] files = listFiles(ftp, fileOrDirectory);
248 for (int i = 0; i < files.length; i++) {
249 String name = files[i].getName();
250 if (".".equals(name) || "..".equals(name)) {
251 continue; // ignore "." and ".."
252 }
253 String file = fileOrDirectory + "/" + name;
254 // This is a file, process it
255 if (!files[i].isDirectory()) {
256 if (getFilter() == null || getFilter().accept(new File(file))) {
257 pollFile(file); // process the file
258 }
259 // Only process directories if processDir is true
260 } else if (processDir) {
261 if (logger.isDebugEnabled()) {
262 logger.debug("Polling directory " + file);
263 }
264 pollFileOrDirectory(ftp, file, isRecursive());
265 } else {
266 if (logger.isDebugEnabled()) {
267 logger.debug("Skipping directory " + file);
268 }
269 }
270 }
271 }
272
273 private FTPFile[] listFiles(FTPClient ftp, String directory) throws IOException {
274 if (changeWorkingDirectory) {
275 ftp.changeWorkingDirectory(directory);
276 return ftp.listFiles();
277 } else {
278 return ftp.listFiles(directory);
279 }
280 }
281
282 protected void pollFile(final String file) {
283 if (logger.isDebugEnabled()) {
284 logger.debug("Scheduling file " + file + " for processing");
285 }
286 getExecutor().execute(new Runnable() {
287 public void run() {
288 final Lock lock = lockManager.getLock(file);
289 if (lock.tryLock()) {
290 processFileNow(file);
291 }
292 }
293 });
294 }
295
296 protected void processFileNow(String file) {
297 FTPClient ftp = null;
298 try {
299 ftp = borrowClient();
300 if (logger.isDebugEnabled()) {
301 logger.debug("Processing file " + file);
302 }
303 if (isFileExistingOnServer(ftp, file)) {
304 // Process the file. If processing fails, an exception should be thrown.
305 processFile(ftp, file);
306 ftp = null;
307 } else {
308 //avoid processing files that have been deleted on the server
309 logger.debug("Skipping " + file + ": the file no longer exists on the server");
310 }
311 } catch (Exception e) {
312 logger.error("Failed to process file: " + file + ". Reason: " + e, e);
313 } finally {
314 if (ftp != null) {
315 returnClient(ftp);
316 }
317 }
318 }
319
320 /**
321 * checks if file specified exists on server
322 *
323 * @param ftp the ftp client
324 * @param file the full file path
325 * @return true if found on server
326 */
327 private boolean isFileExistingOnServer(FTPClient ftp, String file) throws IOException {
328 boolean foundFile = false;
329 int lastIndex = file.lastIndexOf("/");
330 String directory = ".";
331 String rawName = file;
332 if (lastIndex > 0) {
333 directory = file.substring(0, lastIndex);
334 rawName = file.substring(lastIndex+1);
335 }
336
337 FTPFile[] files = listFiles(ftp, directory);
338 if (files.length > 0) {
339 for (FTPFile f : files) {
340 if (f.getName().equals(rawName)) {
341 foundFile = true;
342 break;
343 }
344 }
345 }
346
347 return foundFile;
348 }
349
350 protected void processFile(FTPClient ftp, String file) throws Exception {
351 InputStream in = ftp.retrieveFileStream(file);
352 InOnly exchange = getExchangeFactory().createInOnlyExchange();
353 configureExchangeTarget(exchange);
354 NormalizedMessage message = exchange.createMessage();
355 exchange.setInMessage(message);
356
357 if (getTargetOperation() != null) {
358 exchange.setOperation(getTargetOperation());
359 }
360
361 marshaler.readMessage(exchange, message, in, file);
362 if (stateless) {
363 exchange.setProperty(FtpData.class.getName(), new FtpData(file, ftp, in));
364 } else {
365 this.openExchanges.put(exchange.getExchangeId(), new FtpData(file, ftp, in));
366 }
367 send(exchange);
368 }
369
370 public String getLocationURI() {
371 return uri.toString();
372 }
373
374 public void process(MessageExchange exchange) throws Exception {
375 FtpData data;
376 if (stateless) {
377 data = (FtpData) exchange.getProperty(FtpData.class.getName());
378 } else {
379 data = this.openExchanges.remove(exchange.getExchangeId());
380 }
381 // check for done or error
382 if (data != null) {
383 logger.debug("Releasing " + data.file);
384 try {
385 // Close ftp related stuff
386 data.in.close();
387 data.ftp.completePendingCommand();
388 // check for state
389 if (exchange.getStatus() == ExchangeStatus.DONE) {
390 if (isDeleteFile()) {
391 if (!data.ftp.deleteFile(data.file)) {
392 throw new IOException("Could not delete file " + data.file);
393 }
394 }
395 } else {
396 Exception e = exchange.getError();
397 if (e == null) {
398 e = new JBIException("Unkown error");
399 }
400 throw e;
401 }
402 } finally {
403 // unlock the file
404 unlockAsyncFile(data.file);
405 // release ftp client
406 returnClient(data.ftp);
407 }
408 } else {
409 // strange, we don't know this exchange
410 logger.debug("Received unknown exchange. Will be ignored...");
411 }
412 }
413
414 /**
415 * unlock the file
416 *
417 * @param file the file to unlock
418 */
419 private void unlockAsyncFile(String file) {
420 // finally remove the file from the open exchanges list
421 Lock lock = lockManager.getLock(file);
422 if (lock != null) {
423 try {
424 lock.unlock();
425 } catch (Exception ex) {
426 // can't release the lock
427 logger.error(ex);
428 }
429 }
430 }
431
432 protected FTPClientPool createClientPool() throws Exception {
433 FTPClientPool pool = new FTPClientPool();
434 pool.afterPropertiesSet();
435 return pool;
436 }
437
438 protected FTPClient borrowClient() throws JBIException {
439 try {
440 return (FTPClient) getClientPool().borrowClient();
441 } catch (Exception e) {
442 throw new JBIException(e);
443 }
444 }
445
446 protected void returnClient(FTPClient client) {
447 if (client != null) {
448 try {
449 getClientPool().returnClient(client);
450 } catch (Exception e) {
451 logger.error("Failed to return client to pool: " + e, e);
452 }
453 }
454 }
455
456 }