001 /*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.servicemix.ftp;
018
019 import java.io.File;
020 import java.io.FileFilter;
021 import java.io.IOException;
022 import java.io.InputStream;
023 import java.net.URI;
024 import java.util.concurrent.locks.Lock;
025
026 import javax.jbi.JBIException;
027 import javax.jbi.management.DeploymentException;
028 import javax.jbi.messaging.ExchangeStatus;
029 import javax.jbi.messaging.InOnly;
030 import javax.jbi.messaging.MessageExchange;
031 import javax.jbi.messaging.NormalizedMessage;
032 import javax.jbi.servicedesc.ServiceEndpoint;
033 import javax.xml.namespace.QName;
034
035 import org.apache.commons.net.ftp.FTPClient;
036 import org.apache.commons.net.ftp.FTPFile;
037 import org.apache.servicemix.common.DefaultComponent;
038 import org.apache.servicemix.common.ServiceUnit;
039 import org.apache.servicemix.common.endpoints.PollingEndpoint;
040 import org.apache.servicemix.components.util.DefaultFileMarshaler;
041 import org.apache.servicemix.components.util.FileMarshaler;
042 import org.apache.servicemix.locks.LockManager;
043 import org.apache.servicemix.locks.impl.SimpleLockManager;
044
045 /**
046 * A polling endpoint which looks for a file or files in a directory
047 * and sends the files into the JBI bus as messages, deleting the files
048 * by default when they are processed.
049 *
050 * @org.apache.xbean.XBean element="poller"
051 *
052 * @version $Revision: 468487 $
053 */
054 public class FtpPollerEndpoint extends PollingEndpoint implements FtpEndpointType {
055
056 private FTPClientPool clientPool;
057 private FileFilter filter;
058 private boolean deleteFile = true;
059 private boolean recursive = true;
060 private boolean changeWorkingDirectory;
061 private FileMarshaler marshaler = new DefaultFileMarshaler();
062 private LockManager lockManager;
063 private QName targetOperation;
064 private URI uri;
065
066 public FtpPollerEndpoint() {
067 }
068
069 public FtpPollerEndpoint(ServiceUnit serviceUnit, QName service, String endpoint) {
070 super(serviceUnit, service, endpoint);
071 }
072
073 public FtpPollerEndpoint(DefaultComponent component, ServiceEndpoint endpoint) {
074 super(component, endpoint);
075 }
076
077 public void poll() throws Exception {
078 pollFileOrDirectory(getWorkingPath());
079 }
080
081 public void validate() throws DeploymentException {
082 super.validate();
083 if (uri == null && (getClientPool() == null || getClientPool().getHost() == null)) {
084 throw new DeploymentException("Property uri or clientPool.host must be configured");
085 }
086 if (uri != null && getClientPool() != null && getClientPool().getHost() != null) {
087 throw new DeploymentException("Properties uri and clientPool.host can not be configured at the same time");
088 }
089 if (changeWorkingDirectory && recursive) {
090 throw new DeploymentException("changeWorkingDirectory='true' can not be set when recursive='true'");
091 }
092 }
093
094 public void start() throws Exception {
095 if (lockManager == null) {
096 lockManager = createLockManager();
097 }
098 if (clientPool == null) {
099 clientPool = createClientPool();
100 }
101 if (uri != null) {
102 clientPool.setHost(uri.getHost());
103 clientPool.setPort(uri.getPort());
104 if (uri.getUserInfo() != null) {
105 String[] infos = uri.getUserInfo().split(":");
106 clientPool.setUsername(infos[0]);
107 if (infos.length > 1) {
108 clientPool.setPassword(infos[1]);
109 }
110 }
111 } else {
112 String str = "ftp://" + clientPool.getHost();
113 if (clientPool.getPort() >= 0) {
114 str += ":" + clientPool.getPort();
115 }
116 str += "/";
117 uri = new URI(str);
118 }
119 super.start();
120 }
121
122 protected LockManager createLockManager() {
123 return new SimpleLockManager();
124 }
125
126 private String getWorkingPath() {
127 return (uri != null && uri.getPath() != null) ? uri.getPath() : ".";
128 }
129
130 // Properties
131 //-------------------------------------------------------------------------
132 /**
133 * @return the clientPool
134 */
135 public FTPClientPool getClientPool() {
136 return clientPool;
137 }
138
139 /**
140 * @param clientPool the clientPool to set
141 */
142 public void setClientPool(FTPClientPool clientPool) {
143 this.clientPool = clientPool;
144 }
145
146 /**
147 * @return the uri
148 */
149 public URI getUri() {
150 return uri;
151 }
152
153 /**
154 * @param uri the uri to set
155 */
156 public void setUri(URI uri) {
157 this.uri = uri;
158 }
159
160 public FileFilter getFilter() {
161 return filter;
162 }
163
164 /**
165 * Sets the optional filter to choose which files to process
166 */
167 public void setFilter(FileFilter filter) {
168 this.filter = filter;
169 }
170
171 /**
172 * Returns whether or not we should delete the file when its processed
173 */
174 public boolean isDeleteFile() {
175 return deleteFile;
176 }
177
178 public void setDeleteFile(boolean deleteFile) {
179 this.deleteFile = deleteFile;
180 }
181
182 public boolean isRecursive() {
183 return recursive;
184 }
185
186 public void setRecursive(boolean recursive) {
187 this.recursive = recursive;
188 }
189
190 public FileMarshaler getMarshaler() {
191 return marshaler;
192 }
193
194 public void setMarshaler(FileMarshaler marshaler) {
195 this.marshaler = marshaler;
196 }
197
198 public QName getTargetOperation() { return targetOperation; }
199
200 public void setTargetOperation(QName targetOperation) { this.targetOperation = targetOperation; }
201
202 public void setChangeWorkingDirectory(boolean changeWorkingDirectory) {
203 this.changeWorkingDirectory = changeWorkingDirectory;
204 }
205 // Implementation methods
206 //-------------------------------------------------------------------------
207
208
209 protected void pollFileOrDirectory(String fileOrDirectory) throws Exception {
210 FTPClient ftp = borrowClient();
211 try {
212 logger.debug("Polling directory " + fileOrDirectory);
213 pollFileOrDirectory(ftp, fileOrDirectory, isRecursive());
214 } finally {
215 returnClient(ftp);
216 }
217 }
218
219 protected void pollFileOrDirectory(FTPClient ftp, String fileOrDirectory, boolean processDir) throws Exception {
220 FTPFile[] files = listFiles(ftp, fileOrDirectory);
221 for (int i = 0; i < files.length; i++) {
222 String name = files[i].getName();
223 if (".".equals(name) || "..".equals(name)) {
224 continue; // ignore "." and ".."
225 }
226 String file = fileOrDirectory + "/" + name;
227 // This is a file, process it
228 if (!files[i].isDirectory()) {
229 if (getFilter() == null || getFilter().accept(new File(file))) {
230 pollFile(file); // process the file
231 }
232 // Only process directories if processDir is true
233 } else if (processDir) {
234 if (logger.isDebugEnabled()) {
235 logger.debug("Polling directory " + file);
236 }
237 pollFileOrDirectory(ftp, file, isRecursive());
238 } else {
239 if (logger.isDebugEnabled()) {
240 logger.debug("Skipping directory " + file);
241 }
242 }
243 }
244 }
245
246 private FTPFile[] listFiles(FTPClient ftp, String directory) throws IOException {
247 if (changeWorkingDirectory) {
248 ftp.changeWorkingDirectory(directory);
249 return ftp.listFiles("");
250 } else {
251 return ftp.listFiles(directory);
252 }
253 }
254
255 protected void pollFile(final String file) {
256 if (logger.isDebugEnabled()) {
257 logger.debug("Scheduling file " + file + " for processing");
258 }
259 getExecutor().execute(new Runnable() {
260 public void run() {
261 final Lock lock = lockManager.getLock(file);
262 if (lock.tryLock()) {
263 boolean unlock = true;
264 try {
265 unlock = processFileAndDelete(file);
266 } finally {
267 if (unlock) {
268 try {
269 lock.unlock();
270 } catch (Exception ex) {
271 // can't release the lock
272 logger.error(ex);
273 }
274 lockManager.removeLock(file);
275
276 }
277 }
278 }
279 }
280 });
281 }
282
283 protected boolean processFileAndDelete(String file) {
284 FTPClient ftp = null;
285 boolean unlock = true;
286 try {
287 ftp = borrowClient();
288 if (logger.isDebugEnabled()) {
289 logger.debug("Processing file " + file);
290 }
291 if (isFileExistingOnServer(ftp, file)) {
292 // Process the file. If processing fails, an exception should be thrown.
293 processFile(ftp, file);
294 // Processing is successful
295 // We should not unlock until the file has been deleted
296 unlock = false;
297 if (isDeleteFile()) {
298 if (!ftp.deleteFile(file)) {
299 throw new IOException("Could not delete file " + file);
300 }
301 unlock = true;
302 }
303 } else {
304 //avoid processing files that have been deleted on the server
305 logger.debug("Skipping " + file + ": the file no longer exists on the server");
306 }
307 } catch (Exception e) {
308 logger.error("Failed to process file: " + file + ". Reason: " + e, e);
309 } finally {
310 returnClient(ftp);
311 }
312 return unlock;
313 }
314
315 /**
316 * checks if file specified exists on server
317 *
318 * @param ftp the ftp client
319 * @param file the full file path
320 * @return true if found on server
321 */
322 private boolean isFileExistingOnServer(FTPClient ftp, String file) throws IOException {
323 boolean foundFile = false;
324 int lastIndex = file.lastIndexOf("/");
325 String directory = ".";
326 String rawName = file;
327 if (lastIndex > 0) {
328 directory = file.substring(0, lastIndex);
329 rawName = file.substring(lastIndex + 1);
330 }
331
332 FTPFile[] files = listFiles(ftp, directory);
333 if (files.length > 0) {
334 for (FTPFile f : files) {
335 if (f.getName().equals(rawName)) {
336 foundFile = true;
337 break;
338 }
339 }
340 }
341
342 return foundFile;
343 }
344
345 protected void processFile(FTPClient ftp, String file) throws Exception {
346 InputStream in = ftp.retrieveFileStream(file);
347 InOnly exchange = getExchangeFactory().createInOnlyExchange();
348 configureExchangeTarget(exchange);
349 NormalizedMessage message = exchange.createMessage();
350 exchange.setInMessage(message);
351 if (getTargetOperation() != null) { exchange.setOperation(getTargetOperation()); }
352 marshaler.readMessage(exchange, message, in, file);
353 sendSync(exchange);
354 in.close();
355 ftp.completePendingCommand();
356 if (exchange.getStatus() == ExchangeStatus.ERROR) {
357 Exception e = exchange.getError();
358 if (e == null) {
359 e = new JBIException("Unkown error");
360 }
361 throw e;
362 }
363 }
364
365 public String getLocationURI() {
366 return uri.toString();
367 }
368
369 public void process(MessageExchange exchange) throws Exception {
370 // Do nothing. In our case, this method should never be called
371 // as we only send synchronous InOnly exchange
372 }
373
374 protected FTPClientPool createClientPool() throws Exception {
375 FTPClientPool pool = new FTPClientPool();
376 pool.afterPropertiesSet();
377 return pool;
378 }
379
380 protected FTPClient borrowClient() throws JBIException {
381 try {
382 return (FTPClient) getClientPool().borrowClient();
383 } catch (Exception e) {
384 throw new JBIException(e);
385 }
386 }
387
388 protected void returnClient(FTPClient client) {
389 if (client != null) {
390 try {
391 getClientPool().returnClient(client);
392 } catch (Exception e) {
393 logger.error("Failed to return client to pool: " + e, e);
394 }
395 }
396 }
397
398
399 }