001 /*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.servicemix.ftp;
018
019 import java.io.File;
020 import java.io.FileFilter;
021 import java.io.IOException;
022 import java.io.InputStream;
023 import java.net.URI;
024 import java.util.StringTokenizer;
025 import java.util.concurrent.ConcurrentHashMap;
026 import java.util.concurrent.ConcurrentMap;
027 import java.util.concurrent.locks.Lock;
028
029 import javax.jbi.JBIException;
030 import javax.jbi.management.DeploymentException;
031 import javax.jbi.messaging.ExchangeStatus;
032 import javax.jbi.messaging.InOnly;
033 import javax.jbi.messaging.MessageExchange;
034 import javax.jbi.messaging.NormalizedMessage;
035 import javax.jbi.servicedesc.ServiceEndpoint;
036 import javax.xml.namespace.QName;
037
038 import org.apache.commons.net.ftp.FTPClient;
039 import org.apache.commons.net.ftp.FTPFile;
040 import org.apache.servicemix.common.DefaultComponent;
041 import org.apache.servicemix.common.ServiceUnit;
042 import org.apache.servicemix.common.endpoints.PollingEndpoint;
043 import org.apache.servicemix.common.locks.LockManager;
044 import org.apache.servicemix.common.locks.impl.SimpleLockManager;
045 import org.apache.servicemix.components.util.DefaultFileMarshaler;
046 import org.apache.servicemix.components.util.FileMarshaler;
047
048 /**
049 * A polling endpoint which looks for a file or files in a directory
050 * and sends the files into the JBI bus as messages, deleting the files
051 * by default when they are processed.
052 *
053 * @org.apache.xbean.XBean element="poller"
054 *
055 * @version $Revision: 468487 $
056 */
057 public class FtpPollerEndpoint extends PollingEndpoint implements FtpEndpointType {
058
059 private FTPClientPool clientPool;
060 private FileFilter filter;
061 private boolean deleteFile = true;
062 private boolean recursive = true;
063 private boolean changeWorkingDirectory;
064 private FileMarshaler marshaler = new DefaultFileMarshaler();
065 private LockManager lockManager;
066 private ConcurrentMap<String, FtpData> openExchanges = new ConcurrentHashMap<String, FtpData>();
067 private QName targetOperation;
068 private URI uri;
069 private boolean stateless = true;
070 private URI archive;
071 private boolean autoCreateDirectory = true;
072
073 protected class FtpData {
074 final String file;
075 final FTPClient ftp;
076 final InputStream in;
077 public FtpData(String file, FTPClient ftp, InputStream in) {
078 this.file = file;
079 this.ftp = ftp;
080 this.in = in;
081 }
082 }
083
084 public FtpPollerEndpoint() {
085 }
086
087 public FtpPollerEndpoint(ServiceUnit serviceUnit, QName service, String endpoint) {
088 super(serviceUnit, service, endpoint);
089 }
090
091 public FtpPollerEndpoint(DefaultComponent component, ServiceEndpoint endpoint) {
092 super(component, endpoint);
093 }
094
095 public void poll() throws Exception {
096 pollFileOrDirectory(getWorkingPath());
097 }
098
099 public void validate() throws DeploymentException {
100 super.validate();
101 if (uri == null && (getClientPool() == null || getClientPool().getHost() == null)) {
102 throw new DeploymentException("Property uri or clientPool.host must be configured");
103 }
104 if (uri != null && getClientPool() != null && getClientPool().getHost() != null) {
105 throw new DeploymentException("Properties uri and clientPool.host can not be configured at the same time");
106 }
107 if (changeWorkingDirectory && recursive) {
108 throw new DeploymentException("changeWorkingDirectory='true' can not be set when recursive='true'");
109 }
110 if (archive != null && archive.getPath() == null) {
111 throw new DeploymentException("Archive specified without path information.");
112 }
113 if (archive != null) {
114 if (!deleteFile) {
115 throw new DeploymentException("Archive shouldn't be specified unless deleteFile='true'");
116 }
117 }
118 }
119
120 @Override
121 public synchronized void activate() throws Exception {
122 if (uri == null && clientPool != null) {
123 String str = "ftp://" + clientPool.getHost();
124 if (clientPool.getPort() >= 0) {
125 str += ":" + clientPool.getPort();
126 }
127 str += "/";
128 uri = new URI(str);
129 }
130 super.activate();
131 }
132
133 public void start() throws Exception {
134 if (lockManager == null) {
135 lockManager = createLockManager();
136 }
137 if (clientPool == null) {
138 clientPool = createClientPool();
139 }
140 if (uri != null) {
141 clientPool.setHost(uri.getHost());
142 clientPool.setPort(uri.getPort());
143 if (uri.getUserInfo() != null) {
144 String[] infos = uri.getUserInfo().split(":");
145 clientPool.setUsername(infos[0]);
146 if (infos.length > 1) {
147 clientPool.setPassword(infos[1]);
148 }
149 }
150 }
151
152 // borrow client from pool
153 FTPClient ftp = borrowClient();
154 String folderName = "";
155 try {
156 StringTokenizer strTok = null;
157 if (isAutoCreateDirectory() && !ftp.changeWorkingDirectory(getWorkingPath())) {
158 // it seems the folder isn't there, so create it
159 strTok = new StringTokenizer(getWorkingPath(), "/");
160
161 while (strTok.hasMoreTokens()) {
162 folderName += '/';
163 folderName += strTok.nextToken();
164 if (!ftp.changeWorkingDirectory(folderName)) {
165 if (ftp.makeDirectory(folderName)) {
166 // the folder now exists
167 } else {
168 // unable to create the folder
169 throw new IOException("The defined folder " + getWorkingPath() + " doesn't exist on the server and it can't be created automatically.");
170 }
171 }
172 }
173 }
174 folderName = "";
175 if (getArchivePath() != null) {
176 if (isAutoCreateDirectory() && !ftp.changeWorkingDirectory(getArchivePath())) {
177 // it seems the folder isn't there, so create it
178 strTok = new StringTokenizer(getArchivePath(), "/");
179
180 while (strTok.hasMoreTokens()) {
181 folderName += '/';
182 folderName += strTok.nextToken();
183 if (!ftp.changeWorkingDirectory(folderName)) {
184 if (ftp.makeDirectory(folderName)) {
185 // the folder now exists
186 } else {
187 // unable to create the folder
188 throw new IOException("The defined archive folder " + getArchivePath() + " doesn't exist on the server and it can't be created automatically.");
189 }
190 }
191 }
192 }
193 }
194 } finally {
195 // give back the client
196 returnClient(ftp);
197 }
198
199 super.start();
200 }
201
202 protected LockManager createLockManager() {
203 return new SimpleLockManager();
204 }
205
206 private String getArchivePath() {
207 return (archive != null && archive.getPath() != null) ? archive.getPath() : null;
208 }
209
210 private String getWorkingPath() {
211 return (uri != null && uri.getPath() != null) ? uri.getPath() : ".";
212 }
213
214 // Properties
215 //-------------------------------------------------------------------------
216 /**
217 * @return the clientPool
218 */
219 public FTPClientPool getClientPool() {
220 return clientPool;
221 }
222
223 /**
224 * @param clientPool the clientPool to set
225 */
226 public void setClientPool(FTPClientPool clientPool) {
227 this.clientPool = clientPool;
228 }
229
230 /**
231 * @return the uri
232 */
233 public URI getUri() {
234 return uri;
235 }
236
237 /**
238 * @param uri the uri to set
239 */
240 public void setUri(URI uri) {
241 this.uri = uri;
242 }
243
244 public FileFilter getFilter() {
245 return filter;
246 }
247
248 /**
249 * Sets the optional filter to choose which files to process
250 */
251 public void setFilter(FileFilter filter) {
252 this.filter = filter;
253 }
254
255 /**
256 * Returns whether or not we should delete the file when its processed
257 */
258 public boolean isDeleteFile() {
259 return deleteFile;
260 }
261
262 public void setDeleteFile(boolean deleteFile) {
263 this.deleteFile = deleteFile;
264 }
265
266 public boolean isRecursive() {
267 return recursive;
268 }
269
270 public void setRecursive(boolean recursive) {
271 this.recursive = recursive;
272 }
273
274 public FileMarshaler getMarshaler() {
275 return marshaler;
276 }
277
278 public void setMarshaler(FileMarshaler marshaler) {
279 this.marshaler = marshaler;
280 }
281
282 public QName getTargetOperation() {
283 return targetOperation;
284 }
285
286 public void setTargetOperation(QName targetOperation) {
287 this.targetOperation = targetOperation;
288 }
289
290 public void setChangeWorkingDirectory(boolean changeWorkingDirectory) {
291 this.changeWorkingDirectory = changeWorkingDirectory;
292 }
293
294 public boolean isStateless() {
295 return stateless;
296 }
297
298 public void setStateless(boolean stateless) {
299 this.stateless = stateless;
300 }
301
302 /**
303 * Specifies if the endpoint should create the target directory, if it does
304 * not already exist. If you set this to <code>false</code> and the
305 * directory does not exist, the endpoint will not do anything. Default
306 * value is <code>true</code>.
307 *
308 * @param autoCreateDirectory a boolean specifying if the endpoint creates
309 * directories.
310 */
311 public void setAutoCreateDirectory(boolean autoCreateDirectory) {
312 this.autoCreateDirectory = autoCreateDirectory;
313 }
314
315 public boolean isAutoCreateDirectory() {
316 return autoCreateDirectory;
317 }
318
319 /**
320 * Specifies a directory relative to the polling directory to which
321 * processed files are archived.
322 *
323 * @param archive a <code>URI</code> object for the archive directory
324 */
325 public void setArchive(URI archive) {
326 this.archive = archive;
327 }
328
329 public URI getArchive() {
330 return archive;
331 }
332
333 /**
334 * Bean defining the class implementing the file locking strategy. This bean
335 * must be an implementation of the
336 * <code>org.apache.servicemix.locks.LockManager</code> interface. By
337 * default, this will be set to an instances of
338 * <code>org.apache.servicemix.common.locks.impl.SimpleLockManager</code>.
339 *
340 * @param lockManager the <code>LockManager</code> implementation to use
341 */
342 public void setLockManager(LockManager lockManager) {
343 this.lockManager = lockManager;
344 }
345
346 public LockManager getLockManager() {
347 return lockManager;
348 }
349
350 // Implementation methods
351 //-------------------------------------------------------------------------
352
353 protected void pollFileOrDirectory(String fileOrDirectory) throws Exception {
354 FTPClient ftp = borrowClient();
355 try {
356 logger.debug("Polling directory " + fileOrDirectory);
357 pollFileOrDirectory(ftp, fileOrDirectory, isRecursive());
358 } finally {
359 returnClient(ftp);
360 }
361 }
362
363 protected void pollFileOrDirectory(FTPClient ftp, String fileOrDirectory, boolean processDir) throws Exception {
364 FTPFile[] files = listFiles(ftp, fileOrDirectory);
365 for (int i = 0; i < files.length; i++) {
366 String name = files[i].getName();
367 if (".".equals(name) || "..".equals(name)) {
368 continue; // ignore "." and ".."
369 }
370 String file = fileOrDirectory + "/" + name;
371 // This is a file, process it
372 if (!files[i].isDirectory()) {
373 if (getFilter() == null || getFilter().accept(new File(file))) {
374 pollFile(file); // process the file
375 }
376 // Only process directories if processDir is true
377 } else if (processDir) {
378 if (logger.isDebugEnabled()) {
379 logger.debug("Polling directory " + file);
380 }
381 pollFileOrDirectory(ftp, file, isRecursive());
382 } else {
383 if (logger.isDebugEnabled()) {
384 logger.debug("Skipping directory " + file);
385 }
386 }
387 }
388 }
389
390 private FTPFile[] listFiles(FTPClient ftp, String directory) throws IOException {
391 if (changeWorkingDirectory) {
392 ftp.changeWorkingDirectory(directory);
393 return ftp.listFiles();
394 } else {
395 return ftp.listFiles(directory);
396 }
397 }
398
399 protected void pollFile(final String file) {
400 if (logger.isDebugEnabled()) {
401 logger.debug("Scheduling file " + file + " for processing");
402 }
403 getExecutor().execute(new Runnable() {
404 public void run() {
405 final Lock lock = lockManager.getLock(file);
406 if (lock.tryLock()) {
407 processFileNow(file);
408 }
409 }
410 });
411 }
412
413 protected void processFileNow(String file) {
414 FTPClient ftp = null;
415 try {
416 ftp = borrowClient();
417 if (logger.isDebugEnabled()) {
418 logger.debug("Processing file " + file);
419 }
420 if (isFileExistingOnServer(ftp, file)) {
421 // Process the file. If processing fails, an exception should be thrown.
422 processFile(ftp, file);
423 ftp = null;
424 } else {
425 //avoid processing files that have been deleted on the server
426 logger.debug("Skipping " + file + ": the file no longer exists on the server");
427 }
428 } catch (Exception e) {
429 logger.error("Failed to process file: " + file + ". Reason: " + e, e);
430 } finally {
431 if (ftp != null) {
432 returnClient(ftp);
433 }
434 }
435 }
436
437 /**
438 * checks if file specified exists on server
439 *
440 * @param ftp the ftp client
441 * @param file the full file path
442 * @return true if found on server
443 */
444 private boolean isFileExistingOnServer(FTPClient ftp, String file) throws IOException {
445 boolean foundFile = false;
446 int lastIndex = file.lastIndexOf("/");
447 String directory = ".";
448 String rawName = file;
449 if (lastIndex > 0) {
450 directory = file.substring(0, lastIndex);
451 rawName = file.substring(lastIndex+1);
452 }
453
454 FTPFile[] files = listFiles(ftp, directory);
455 if (files.length > 0) {
456 for (FTPFile f : files) {
457 if (f.getName().equals(rawName)) {
458 foundFile = true;
459 break;
460 }
461 }
462 }
463
464 return foundFile;
465 }
466
467 protected void processFile(FTPClient ftp, String file) throws Exception {
468 InputStream in = ftp.retrieveFileStream(file);
469 InOnly exchange = getExchangeFactory().createInOnlyExchange();
470 configureExchangeTarget(exchange);
471 NormalizedMessage message = exchange.createMessage();
472 exchange.setInMessage(message);
473
474 if (getTargetOperation() != null) {
475 exchange.setOperation(getTargetOperation());
476 }
477
478 marshaler.readMessage(exchange, message, in, file);
479 if (stateless) {
480 exchange.setProperty(FtpData.class.getName(), new FtpData(file, ftp, in));
481 } else {
482 this.openExchanges.put(exchange.getExchangeId(), new FtpData(file, ftp, in));
483 }
484 send(exchange);
485 }
486
487 public String getLocationURI() {
488 return uri.toString();
489 }
490
491 public void process(MessageExchange exchange) throws Exception {
492 FtpData data;
493 if (stateless) {
494 data = (FtpData) exchange.getProperty(FtpData.class.getName());
495 } else {
496 data = this.openExchanges.remove(exchange.getExchangeId());
497 }
498 // check for done or error
499 if (data != null) {
500 logger.debug("Releasing " + data.file);
501 try {
502 // Close ftp related stuff
503 data.in.close();
504 data.ftp.completePendingCommand();
505 // check for state
506 if (exchange.getStatus() == ExchangeStatus.DONE) {
507 if (isDeleteFile()) {
508 if (getArchivePath() != null) {
509 // build a unique archive file name
510 String newPath = String.format("%s/%d_%s", getArchivePath(), System.currentTimeMillis(), data.file.substring(data.file.lastIndexOf('/')+1));
511 data.ftp.rename(data.file, newPath);
512 } else {
513 if (!data.ftp.deleteFile(data.file)) {
514 throw new IOException("Could not delete file " + data.file);
515 }
516 }
517 }
518 } else {
519 Exception e = exchange.getError();
520 if (e == null) {
521 e = new JBIException("Unknown error");
522 }
523 throw e;
524 }
525 } finally {
526 // unlock the file
527 unlockAsyncFile(data.file);
528 // release ftp client
529 returnClient(data.ftp);
530 }
531 } else {
532 // strange, we don't know this exchange
533 logger.debug("Received unknown exchange. Will be ignored...");
534 }
535 }
536
537 /**
538 * unlock the file
539 *
540 * @param file the file to unlock
541 */
542 private void unlockAsyncFile(String file) {
543 // finally remove the file from the open exchanges list
544 Lock lock = lockManager.getLock(file);
545 if (lock != null) {
546 try {
547 lock.unlock();
548 } catch (Exception ex) {
549 // can't release the lock
550 logger.error(ex);
551 }
552 lockManager.removeLock(file);
553 }
554 }
555
556 protected FTPClientPool createClientPool() throws Exception {
557 FTPClientPool pool = new FTPClientPool();
558 pool.afterPropertiesSet();
559 return pool;
560 }
561
562 protected FTPClient borrowClient() throws JBIException {
563 try {
564 return (FTPClient) getClientPool().borrowClient();
565 } catch (Exception e) {
566 throw new JBIException(e);
567 }
568 }
569
570 protected void returnClient(FTPClient client) {
571 if (client != null) {
572 try {
573 getClientPool().returnClient(client);
574 } catch (Exception e) {
575 logger.error("Failed to return client to pool: " + e, e);
576 }
577 }
578 }
579
580 }