001 /*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.servicemix.ftp;
018
019 import java.io.File;
020 import java.io.FileFilter;
021 import java.io.IOException;
022 import java.io.InputStream;
023 import java.net.URI;
024 import java.util.concurrent.locks.Lock;
025
026 import javax.jbi.JBIException;
027 import javax.jbi.management.DeploymentException;
028 import javax.jbi.messaging.ExchangeStatus;
029 import javax.jbi.messaging.InOnly;
030 import javax.jbi.messaging.MessageExchange;
031 import javax.jbi.messaging.NormalizedMessage;
032 import javax.jbi.servicedesc.ServiceEndpoint;
033 import javax.xml.namespace.QName;
034
035 import org.apache.commons.net.ftp.FTPClient;
036 import org.apache.commons.net.ftp.FTPFile;
037 import org.apache.servicemix.common.DefaultComponent;
038 import org.apache.servicemix.common.ServiceUnit;
039 import org.apache.servicemix.common.endpoints.PollingEndpoint;
040 import org.apache.servicemix.components.util.DefaultFileMarshaler;
041 import org.apache.servicemix.components.util.FileMarshaler;
042 import org.apache.servicemix.locks.LockManager;
043 import org.apache.servicemix.locks.impl.SimpleLockManager;
044
045 /**
046 * A polling endpoint which looks for a file or files in a directory
047 * and sends the files into the JBI bus as messages, deleting the files
048 * by default when they are processed.
049 *
050 * @org.apache.xbean.XBean element="poller"
051 *
052 * @version $Revision: 468487 $
053 */
054 public class FtpPollerEndpoint extends PollingEndpoint implements FtpEndpointType {
055
056 private FTPClientPool clientPool;
057 private FileFilter filter;
058 private boolean deleteFile = true;
059 private boolean recursive = true;
060 private boolean changeWorkingDirectory;
061 private FileMarshaler marshaler = new DefaultFileMarshaler();
062 private LockManager lockManager;
063 private QName targetOperation;
064 private URI uri;
065
066 public FtpPollerEndpoint() {
067 }
068
069 public FtpPollerEndpoint(ServiceUnit serviceUnit, QName service, String endpoint) {
070 super(serviceUnit, service, endpoint);
071 }
072
073 public FtpPollerEndpoint(DefaultComponent component, ServiceEndpoint endpoint) {
074 super(component, endpoint);
075 }
076
077 public void poll() throws Exception {
078 pollFileOrDirectory(getWorkingPath());
079 }
080
081 public void validate() throws DeploymentException {
082 super.validate();
083 if (uri == null && (getClientPool() == null || getClientPool().getHost() == null)) {
084 throw new DeploymentException("Property uri or clientPool.host must be configured");
085 }
086 if (uri != null && getClientPool() != null && getClientPool().getHost() != null) {
087 throw new DeploymentException("Properties uri and clientPool.host can not be configured at the same time");
088 }
089 if (changeWorkingDirectory && recursive) {
090 throw new DeploymentException("changeWorkingDirectory='true' can not be set when recursive='true'");
091 }
092 }
093
094 public void start() throws Exception {
095 if (lockManager == null) {
096 lockManager = createLockManager();
097 }
098 if (clientPool == null) {
099 clientPool = createClientPool();
100 }
101 if (uri != null) {
102 clientPool.setHost(uri.getHost());
103 clientPool.setPort(uri.getPort());
104 if (uri.getUserInfo() != null) {
105 String[] infos = uri.getUserInfo().split(":");
106 clientPool.setUsername(infos[0]);
107 if (infos.length > 1) {
108 clientPool.setPassword(infos[1]);
109 }
110 }
111 } else {
112 String str = "ftp://" + clientPool.getHost();
113 if (clientPool.getPort() >= 0) {
114 str += ":" + clientPool.getPort();
115 }
116 str += "/";
117 uri = new URI(str);
118 }
119 super.start();
120 }
121
122 protected LockManager createLockManager() {
123 return new SimpleLockManager();
124 }
125
126 private String getWorkingPath() {
127 return (uri != null && uri.getPath() != null) ? uri.getPath() : ".";
128 }
129
130 // Properties
131 //-------------------------------------------------------------------------
132 /**
133 * @return the clientPool
134 */
135 public FTPClientPool getClientPool() {
136 return clientPool;
137 }
138
139 /**
140 * @param clientPool the clientPool to set
141 */
142 public void setClientPool(FTPClientPool clientPool) {
143 this.clientPool = clientPool;
144 }
145
146 /**
147 * @return the uri
148 */
149 public URI getUri() {
150 return uri;
151 }
152
153 /**
154 * @param uri the uri to set
155 */
156 public void setUri(URI uri) {
157 this.uri = uri;
158 }
159
160 public FileFilter getFilter() {
161 return filter;
162 }
163
164 /**
165 * Sets the optional filter to choose which files to process
166 */
167 public void setFilter(FileFilter filter) {
168 this.filter = filter;
169 }
170
171 /**
172 * Returns whether or not we should delete the file when its processed
173 */
174 public boolean isDeleteFile() {
175 return deleteFile;
176 }
177
178 public void setDeleteFile(boolean deleteFile) {
179 this.deleteFile = deleteFile;
180 }
181
182 public boolean isRecursive() {
183 return recursive;
184 }
185
186 public void setRecursive(boolean recursive) {
187 this.recursive = recursive;
188 }
189
190 public FileMarshaler getMarshaler() {
191 return marshaler;
192 }
193
194 public void setMarshaler(FileMarshaler marshaler) {
195 this.marshaler = marshaler;
196 }
197
198 public QName getTargetOperation() { return targetOperation; }
199
200 public void setTargetOperation(QName targetOperation) { this.targetOperation = targetOperation; }
201
202 public void setChangeWorkingDirectory(boolean changeWorkingDirectory) {
203 this.changeWorkingDirectory = changeWorkingDirectory;
204 }
205 // Implementation methods
206 //-------------------------------------------------------------------------
207
208
209 protected void pollFileOrDirectory(String fileOrDirectory) throws Exception {
210 FTPClient ftp = borrowClient();
211 try {
212 logger.debug("Polling directory " + fileOrDirectory);
213 pollFileOrDirectory(ftp, fileOrDirectory, isRecursive());
214 } finally {
215 returnClient(ftp);
216 }
217 }
218
219 protected void pollFileOrDirectory(FTPClient ftp, String fileOrDirectory, boolean processDir) throws Exception {
220 FTPFile[] files = listFiles(ftp, fileOrDirectory);
221 for (int i = 0; i < files.length; i++) {
222 String name = files[i].getName();
223 if (".".equals(name) || "..".equals(name)) {
224 continue; // ignore "." and ".."
225 }
226 String file = fileOrDirectory + "/" + name;
227 // This is a file, process it
228 if (!files[i].isDirectory()) {
229 if (getFilter() == null || getFilter().accept(new File(file))) {
230 pollFile(file); // process the file
231 }
232 // Only process directories if processDir is true
233 } else if (processDir) {
234 if (logger.isDebugEnabled()) {
235 logger.debug("Polling directory " + file);
236 }
237 pollFileOrDirectory(ftp, file, isRecursive());
238 } else {
239 if (logger.isDebugEnabled()) {
240 logger.debug("Skipping directory " + file);
241 }
242 }
243 }
244 }
245
246 private FTPFile[] listFiles(FTPClient ftp, String directory) throws IOException {
247 if (changeWorkingDirectory) {
248 ftp.changeWorkingDirectory(directory);
249 return ftp.listFiles("");
250 } else {
251 return ftp.listFiles(directory);
252 }
253 }
254
255 protected void pollFile(final String file) {
256 if (logger.isDebugEnabled()) {
257 logger.debug("Scheduling file " + file + " for processing");
258 }
259 getExecutor().execute(new Runnable() {
260 public void run() {
261 final Lock lock = lockManager.getLock(file);
262 if (lock.tryLock()) {
263 boolean unlock = true;
264 try {
265 unlock = processFileAndDelete(file);
266 } finally {
267 if (unlock) {
268 lock.unlock();
269 }
270 }
271 }
272 }
273 });
274 }
275
276 protected boolean processFileAndDelete(String file) {
277 FTPClient ftp = null;
278 boolean unlock = true;
279 try {
280 ftp = borrowClient();
281 if (logger.isDebugEnabled()) {
282 logger.debug("Processing file " + file);
283 }
284 if (ftp.listFiles(file).length > 0) {
285 // Process the file. If processing fails, an exception should be thrown.
286 processFile(ftp, file);
287 // Processing is successful
288 // We should not unlock until the file has been deleted
289 unlock = false;
290 if (isDeleteFile()) {
291 if (!ftp.deleteFile(file)) {
292 throw new IOException("Could not delete file " + file);
293 }
294 unlock = true;
295 }
296 } else {
297 //avoid processing files that have been deleted on the server
298 logger.debug("Skipping " + file + ": the file no longer exists on the server");
299 }
300 } catch (Exception e) {
301 logger.error("Failed to process file: " + file + ". Reason: " + e, e);
302 } finally {
303 returnClient(ftp);
304 }
305 return unlock;
306 }
307
308 protected void processFile(FTPClient ftp, String file) throws Exception {
309 InputStream in = ftp.retrieveFileStream(file);
310 InOnly exchange = getExchangeFactory().createInOnlyExchange();
311 configureExchangeTarget(exchange);
312 NormalizedMessage message = exchange.createMessage();
313 exchange.setInMessage(message);
314 if (getTargetOperation() != null) { exchange.setOperation(getTargetOperation()); }
315 marshaler.readMessage(exchange, message, in, file);
316 sendSync(exchange);
317 in.close();
318 ftp.completePendingCommand();
319 if (exchange.getStatus() == ExchangeStatus.ERROR) {
320 Exception e = exchange.getError();
321 if (e == null) {
322 e = new JBIException("Unkown error");
323 }
324 throw e;
325 }
326 }
327
328 public String getLocationURI() {
329 return uri.toString();
330 }
331
332 public void process(MessageExchange exchange) throws Exception {
333 // Do nothing. In our case, this method should never be called
334 // as we only send synchronous InOnly exchange
335 }
336
337 protected FTPClientPool createClientPool() throws Exception {
338 FTPClientPool pool = new FTPClientPool();
339 pool.afterPropertiesSet();
340 return pool;
341 }
342
343 protected FTPClient borrowClient() throws JBIException {
344 try {
345 return (FTPClient) getClientPool().borrowClient();
346 } catch (Exception e) {
347 throw new JBIException(e);
348 }
349 }
350
351 protected void returnClient(FTPClient client) {
352 if (client != null) {
353 try {
354 getClientPool().returnClient(client);
355 } catch (Exception e) {
356 logger.error("Failed to return client to pool: " + e, e);
357 }
358 }
359 }
360
361
362 }