001 /*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.servicemix.ftp;
018
019 import java.io.File;
020 import java.io.FileFilter;
021 import java.io.IOException;
022 import java.io.InputStream;
023 import java.net.URI;
024 import java.util.concurrent.locks.Lock;
025
026 import javax.jbi.JBIException;
027 import javax.jbi.management.DeploymentException;
028 import javax.jbi.messaging.ExchangeStatus;
029 import javax.jbi.messaging.InOnly;
030 import javax.jbi.messaging.MessageExchange;
031 import javax.jbi.messaging.NormalizedMessage;
032 import javax.jbi.servicedesc.ServiceEndpoint;
033 import javax.xml.namespace.QName;
034
035 import org.apache.commons.net.ftp.FTPClient;
036 import org.apache.commons.net.ftp.FTPFile;
037 import org.apache.servicemix.common.DefaultComponent;
038 import org.apache.servicemix.common.ServiceUnit;
039 import org.apache.servicemix.common.endpoints.PollingEndpoint;
040 import org.apache.servicemix.components.util.DefaultFileMarshaler;
041 import org.apache.servicemix.components.util.FileMarshaler;
042 import org.apache.servicemix.locks.LockManager;
043 import org.apache.servicemix.locks.impl.SimpleLockManager;
044
045 /**
046 * A polling endpoint which looks for a file or files in a directory
047 * and sends the files into the JBI bus as messages, deleting the files
048 * by default when they are processed.
049 *
050 * @org.apache.xbean.XBean element="poller"
051 *
052 * @version $Revision: 468487 $
053 */
054 public class FtpPollerEndpoint extends PollingEndpoint implements FtpEndpointType {
055
056 private FTPClientPool clientPool;
057 private FileFilter filter;
058 private boolean deleteFile = true;
059 private boolean recursive = true;
060 private FileMarshaler marshaler = new DefaultFileMarshaler();
061 private LockManager lockManager;
062 private QName targetOperation;
063 private URI uri;
064
065 public FtpPollerEndpoint() {
066 }
067
068 public FtpPollerEndpoint(ServiceUnit serviceUnit, QName service, String endpoint) {
069 super(serviceUnit, service, endpoint);
070 }
071
072 public FtpPollerEndpoint(DefaultComponent component, ServiceEndpoint endpoint) {
073 super(component, endpoint);
074 }
075
076 public void poll() throws Exception {
077 pollFileOrDirectory(getWorkingPath());
078 }
079
080 public void validate() throws DeploymentException {
081 super.validate();
082 if (uri == null && (getClientPool() == null || getClientPool().getHost() == null)) {
083 throw new DeploymentException("Property uri or clientPool.host must be configured");
084 }
085 if (uri != null && getClientPool() != null && getClientPool().getHost() != null) {
086 throw new DeploymentException("Properties uri and clientPool.host can not be configured at the same time");
087 }
088 }
089
090 public void start() throws Exception {
091 if (lockManager == null) {
092 lockManager = createLockManager();
093 }
094 if (clientPool == null) {
095 clientPool = createClientPool();
096 }
097 if (uri != null) {
098 clientPool.setHost(uri.getHost());
099 clientPool.setPort(uri.getPort());
100 if (uri.getUserInfo() != null) {
101 String[] infos = uri.getUserInfo().split(":");
102 clientPool.setUsername(infos[0]);
103 if (infos.length > 1) {
104 clientPool.setPassword(infos[1]);
105 }
106 }
107 } else {
108 String str = "ftp://" + clientPool.getHost();
109 if (clientPool.getPort() >= 0) {
110 str += ":" + clientPool.getPort();
111 }
112 str += "/";
113 uri = new URI(str);
114 }
115 super.start();
116 }
117
118 protected LockManager createLockManager() {
119 return new SimpleLockManager();
120 }
121
122 private String getWorkingPath() {
123 return (uri != null && uri.getPath() != null) ? uri.getPath() : ".";
124 }
125
126 // Properties
127 //-------------------------------------------------------------------------
128 /**
129 * @return the clientPool
130 */
131 public FTPClientPool getClientPool() {
132 return clientPool;
133 }
134
135 /**
136 * @param clientPool the clientPool to set
137 */
138 public void setClientPool(FTPClientPool clientPool) {
139 this.clientPool = clientPool;
140 }
141
142 /**
143 * @return the uri
144 */
145 public URI getUri() {
146 return uri;
147 }
148
149 /**
150 * @param uri the uri to set
151 */
152 public void setUri(URI uri) {
153 this.uri = uri;
154 }
155
156 public FileFilter getFilter() {
157 return filter;
158 }
159
160 /**
161 * Sets the optional filter to choose which files to process
162 */
163 public void setFilter(FileFilter filter) {
164 this.filter = filter;
165 }
166
167 /**
168 * Returns whether or not we should delete the file when its processed
169 */
170 public boolean isDeleteFile() {
171 return deleteFile;
172 }
173
174 public void setDeleteFile(boolean deleteFile) {
175 this.deleteFile = deleteFile;
176 }
177
178 public boolean isRecursive() {
179 return recursive;
180 }
181
182 public void setRecursive(boolean recursive) {
183 this.recursive = recursive;
184 }
185
186 public FileMarshaler getMarshaler() {
187 return marshaler;
188 }
189
190 public void setMarshaler(FileMarshaler marshaler) {
191 this.marshaler = marshaler;
192 }
193
194 public QName getTargetOperation() { return targetOperation; }
195
196 public void setTargetOperation(QName targetOperation) { this.targetOperation = targetOperation; }
197
198 // Implementation methods
199 //-------------------------------------------------------------------------
200
201
202 protected void pollFileOrDirectory(String fileOrDirectory) throws Exception {
203 FTPClient ftp = borrowClient();
204 try {
205 logger.debug("Polling directory " + fileOrDirectory);
206 pollFileOrDirectory(ftp, fileOrDirectory, isRecursive());
207 } finally {
208 returnClient(ftp);
209 }
210 }
211
212 protected void pollFileOrDirectory(FTPClient ftp, String fileOrDirectory, boolean processDir) throws Exception {
213 FTPFile[] files = ftp.listFiles(fileOrDirectory);
214 for (int i = 0; i < files.length; i++) {
215 String name = files[i].getName();
216 if (".".equals(name) || "..".equals(name)) {
217 continue; // ignore "." and ".."
218 }
219 String file = fileOrDirectory + "/" + name;
220 // This is a file, process it
221 if (!files[i].isDirectory()) {
222 if (getFilter() == null || getFilter().accept(new File(file))) {
223 pollFile(file); // process the file
224 }
225 // Only process directories if processDir is true
226 } else if (processDir) {
227 if (logger.isDebugEnabled()) {
228 logger.debug("Polling directory " + file);
229 }
230 pollFileOrDirectory(ftp, file, isRecursive());
231 } else {
232 if (logger.isDebugEnabled()) {
233 logger.debug("Skipping directory " + file);
234 }
235 }
236 }
237 }
238
239 protected void pollFile(final String file) {
240 if (logger.isDebugEnabled()) {
241 logger.debug("Scheduling file " + file + " for processing");
242 }
243 getExecutor().execute(new Runnable() {
244 public void run() {
245 final Lock lock = lockManager.getLock(file);
246 if (lock.tryLock()) {
247 boolean unlock = true;
248 try {
249 unlock = processFileAndDelete(file);
250 } finally {
251 if (unlock) {
252 lock.unlock();
253 }
254 }
255 }
256 }
257 });
258 }
259
260 protected boolean processFileAndDelete(String file) {
261 FTPClient ftp = null;
262 boolean unlock = true;
263 try {
264 ftp = borrowClient();
265 if (logger.isDebugEnabled()) {
266 logger.debug("Processing file " + file);
267 }
268 if (ftp.listFiles(file).length > 0) {
269 // Process the file. If processing fails, an exception should be thrown.
270 processFile(ftp, file);
271 // Processing is successful
272 // We should not unlock until the file has been deleted
273 unlock = false;
274 if (isDeleteFile()) {
275 if (!ftp.deleteFile(file)) {
276 throw new IOException("Could not delete file " + file);
277 }
278 unlock = true;
279 }
280 } else {
281 //avoid processing files that have been deleted on the server
282 logger.debug("Skipping " + file + ": the file no longer exists on the server");
283 }
284 } catch (Exception e) {
285 logger.error("Failed to process file: " + file + ". Reason: " + e, e);
286 } finally {
287 returnClient(ftp);
288 }
289 return unlock;
290 }
291
292 protected void processFile(FTPClient ftp, String file) throws Exception {
293 InputStream in = ftp.retrieveFileStream(file);
294 InOnly exchange = getExchangeFactory().createInOnlyExchange();
295 configureExchangeTarget(exchange);
296 NormalizedMessage message = exchange.createMessage();
297 exchange.setInMessage(message);
298 if (getTargetOperation() != null) { exchange.setOperation(getTargetOperation()); }
299 marshaler.readMessage(exchange, message, in, file);
300 sendSync(exchange);
301 in.close();
302 ftp.completePendingCommand();
303 if (exchange.getStatus() == ExchangeStatus.ERROR) {
304 Exception e = exchange.getError();
305 if (e == null) {
306 e = new JBIException("Unkown error");
307 }
308 throw e;
309 }
310 }
311
312 public String getLocationURI() {
313 return uri.toString();
314 }
315
316 public void process(MessageExchange exchange) throws Exception {
317 // Do nothing. In our case, this method should never be called
318 // as we only send synchronous InOnly exchange
319 }
320
321 protected FTPClientPool createClientPool() throws Exception {
322 FTPClientPool pool = new FTPClientPool();
323 pool.afterPropertiesSet();
324 return pool;
325 }
326
327 protected FTPClient borrowClient() throws JBIException {
328 try {
329 return (FTPClient) getClientPool().borrowClient();
330 } catch (Exception e) {
331 throw new JBIException(e);
332 }
333 }
334
335 protected void returnClient(FTPClient client) {
336 if (client != null) {
337 try {
338 getClientPool().returnClient(client);
339 } catch (Exception e) {
340 logger.error("Failed to return client to pool: " + e, e);
341 }
342 }
343 }
344
345 }