001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016 package org.apache.servicemix.vfs;
017
018 import java.io.IOException;
019 import java.io.InputStream;
020 import java.util.Arrays;
021 import java.util.Comparator;
022 import java.util.Set;
023 import java.util.concurrent.ConcurrentHashMap;
024 import java.util.concurrent.ConcurrentMap;
025 import java.util.concurrent.CopyOnWriteArraySet;
026 import java.util.concurrent.locks.Lock;
027
028 import javax.jbi.JBIException;
029 import javax.jbi.messaging.ExchangeStatus;
030 import javax.jbi.messaging.InOnly;
031 import javax.jbi.messaging.MessageExchange;
032 import javax.jbi.messaging.NormalizedMessage;
033 import javax.jbi.servicedesc.ServiceEndpoint;
034 import javax.xml.namespace.QName;
035
036 import org.apache.commons.logging.Log;
037 import org.apache.commons.logging.LogFactory;
038 import org.apache.commons.vfs.FileContent;
039 import org.apache.commons.vfs.FileObject;
040 import org.apache.commons.vfs.FileSelector;
041 import org.apache.commons.vfs.FileSystemManager;
042 import org.apache.commons.vfs.FileType;
043 import org.apache.servicemix.common.DefaultComponent;
044 import org.apache.servicemix.common.ServiceUnit;
045 import org.apache.servicemix.common.endpoints.PollingEndpoint;
046 import org.apache.servicemix.common.locks.LockManager;
047 import org.apache.servicemix.common.locks.impl.SimpleLockManager;
048 import org.apache.servicemix.components.util.DefaultFileMarshaler;
049 import org.apache.servicemix.components.util.FileMarshaler;
050 import org.apache.servicemix.executors.ExecutorAwareRunnable;
051
052 /**
053 * A polling endpoint that looks for a file or files in a virtual file system
054 * and sends the files to a target service (via the JBI bus), deleting the files
055 * by default when they are processed. The polling endpoint uses a file marshaler
056 * to send the data as a JBI message; by default this marshaler expects XML
057 * payload. For non-XML payload, e.g. plain-text or binary files, use an
058 * alternative marshaler such as the
059 * <code>org.apache.servicemix.components.util.BinaryFileMarshaler</code>
060 *
061 * @org.apache.xbean.XBean element="poller"
062 *
063 * @author lhein
064 */
065 public class VFSPollingEndpoint extends PollingEndpoint implements VFSEndpointType {
066 private static final Log logger = LogFactory.getLog(VFSPollingEndpoint.class);
067
068 private FileMarshaler marshaler = new DefaultFileMarshaler();
069 private FileObject file;
070 private FileSelector selector;
071 private Set<FileObject> workingSet = new CopyOnWriteArraySet<FileObject>();
072 private boolean deleteFile =true;
073 private boolean recursive = true;
074 private String path;
075 private Comparator<FileObject> comparator = null;
076 private FileSystemManager fileSystemManager;
077 private LockManager lockManager;
078 private ConcurrentMap<String, InputStream> openExchanges = new ConcurrentHashMap<String, InputStream>();
079 private boolean concurrentExchange = true;
080 /**
081 * default constructor
082 */
083 public VFSPollingEndpoint() {
084 }
085
086 /**
087 * creates a VFS polling endpoint
088 *
089 * @param serviceUnit the service unit
090 * @param service the service name
091 * @param endpoint the endpoint name
092 */
093 public VFSPollingEndpoint(ServiceUnit serviceUnit, QName service, String endpoint) {
094 super(serviceUnit, service, endpoint);
095 }
096
097 /**
098 * creates a VFS polling endpoint
099 *
100 * @param component the default component
101 * @param endpoint the endpoint
102 */
103 public VFSPollingEndpoint(DefaultComponent component, ServiceEndpoint endpoint) {
104 super(component, endpoint);
105 }
106
107 /* (non-Javadoc)
108 * @see org.apache.servicemix.common.endpoints.PollingEndpoint#start()
109 */
110 @Override
111 public synchronized void start() throws Exception {
112 super.start();
113
114 // clear the set of already processed files
115 this.workingSet.clear();
116
117 // re-create the openExchanges map
118 this.openExchanges = new ConcurrentHashMap<String, InputStream>();
119
120 // create a lock manager
121 if (lockManager == null) {
122 lockManager = createLockManager();
123 }
124 }
125
126 /**
127 * returns the lock manager
128 *
129 * @return the lock manager
130 */
131 protected LockManager createLockManager() {
132 return new SimpleLockManager();
133 }
134
135 /*
136 * (non-Javadoc)
137 * @see org.apache.servicemix.common.endpoints.ConsumerEndpoint#getLocationURI()
138 */
139 @Override
140 public String getLocationURI() {
141 // return a URI that unique identify this endpoint
142 return getService() + "#" + getEndpoint();
143 }
144
145 /* (non-Javadoc)
146 * @see org.apache.servicemix.common.endpoints.AbstractEndpoint#process(javax.jbi.messaging.MessageExchange)
147 */
148 @Override
149 public void process(MessageExchange exchange) throws Exception {
150 // check for done or error
151 if (this.openExchanges.containsKey(exchange.getExchangeId())) {
152 InputStream stream = this.openExchanges.get(exchange.getExchangeId());
153 FileObject aFile = (FileObject)exchange.getMessage("in").getProperty(VFSComponent.VFS_PROPERTY);
154
155 if (aFile == null) {
156 throw new JBIException(
157 "Property org.apache.servicemix.vfs was removed from the exchange -- unable to delete/archive the file");
158 }
159
160 logger.debug("Releasing " + aFile.getName().getPathDecoded());
161
162 // first try to close the stream
163 try {
164 stream.close();
165 } catch (IOException ex) {
166 logger.error("Unable to close stream for file " + aFile.getName().getPathDecoded(), ex);
167 }
168
169 try {
170 // check for state
171 if (exchange.getStatus() == ExchangeStatus.DONE) {
172 if (isDeleteFile()) {
173 if (!aFile.delete()) {
174 throw new IOException("Could not delete file " + aFile.getName().getPathDecoded());
175 }
176 }
177 } else if (exchange.getStatus() == ExchangeStatus.ERROR) {
178 Exception e = exchange.getError();
179 if (e == null) {
180 throw new JBIException(
181 "Received an exchange with status ERROR, but no exception was set");
182 }
183 logger.warn("Message in file " + aFile.getName().getPathDecoded() + " could not be handled successfully: "
184 + e.getMessage(), e);
185 } else {
186 // we should never get an ACTIVE exchange -- the File poller
187 // only sends InOnly exchanges
188 throw new JBIException("Unexpectedly received an exchange with status ACTIVE");
189 }
190 } finally {
191 // remove file from set of already processed files
192 workingSet.remove(aFile);
193 // remove the open exchange
194 openExchanges.remove(exchange.getExchangeId());
195 // unlock the file
196 unlockAsyncFile(aFile);
197 }
198 } else {
199 // strange, we don't know this exchange
200 logger.debug("Received unknown exchange. Will be ignored...");
201 return;
202 }
203 }
204
205 /**
206 * unlock the file
207 *
208 * @param file the file to unlock
209 */
210 private void unlockAsyncFile(FileObject file) {
211 // finally remove the file from the open exchanges list
212 String uri = file.getName().getURI().toString();
213 Lock lock = lockManager.getLock(uri);
214 if (lock != null) {
215 try {
216 lock.unlock();
217 } catch (Exception ex) {
218 // can't release the lock
219 logger.error(ex);
220 }
221 }
222 }
223
224 /* (non-Javadoc)
225 * @see org.apache.servicemix.common.endpoints.PollingEndpoint#poll()
226 */
227 @Override
228 public void poll() throws Exception {
229 // resolve the path to a FileObject
230 if (file == null) {
231 try {
232 file = FileObjectResolver.resolveToFileObject(getFileSystemManager(), getPath());
233 } catch (Exception e) {
234 logger.debug("Unable to resolve path: " + getPath(), e);
235 file = null;
236 }
237 }
238
239 // SM-192: Force close the file, so that the cached informations are cleared
240 if (file != null) {
241 file.close();
242 pollFileOrDirectory(file);
243 }
244 }
245
246 /**
247 * polls a file which is not clear to be a file or folder
248 *
249 * @param fileOrDirectory the file or folder object
250 * @throws Exception on IO errors
251 */
252 protected void pollFileOrDirectory(FileObject fileOrDirectory) throws Exception {
253 pollFileOrDirectory(fileOrDirectory, true);
254 }
255
256 /**
257 * recursive method for processing a file or a folder
258 *
259 * @param fileOrDirectory the file or folder object
260 * @param processDir flag if processing should act recursive
261 * @throws Exception on IO errors
262 */
263 protected void pollFileOrDirectory(FileObject fileOrDirectory, boolean processDir) throws Exception {
264 // check if it is a file object
265 if (fileOrDirectory.getType().equals(FileType.FILE)) {
266 // process the file
267 pollFile(fileOrDirectory);
268 } else if (processDir) {
269 // process the folder
270 logger.debug("Polling directory " + fileOrDirectory.getName().getPathDecoded());
271
272 FileObject[] files = null;
273 if (selector != null) {
274 files = sortPolledFiles(fileOrDirectory.findFiles(selector));
275 } else {
276 files = sortPolledFiles(fileOrDirectory.getChildren());
277 }
278 // process each file inside folder
279 for (FileObject f : files) {
280 // self-recursion
281 pollFileOrDirectory(f, isRecursive());
282 }
283 } else {
284 logger.debug("Skipping directory " + fileOrDirectory.getName().getPathDecoded());
285 }
286 }
287
288 /**
289 * sorts polled file using the given comparator. If the comparator is null, no change are applied on polled files order.
290 *
291 * @param files the polled file object array.
292 * @return the sorted polled file object array.
293 */
294 private FileObject[] sortPolledFiles(FileObject[] files) {
295 if (comparator == null) {
296 return files;
297 }
298 Arrays.sort(files, comparator);
299 return files;
300 }
301
302 /**
303 * polls a file object
304 *
305 * @param aFile the file object
306 * @throws Exception on IO errors
307 */
308 protected void pollFile(final FileObject aFile) throws Exception {
309 // check if file is fully available
310 if (!isFullyAvailable(aFile)) {
311 return;
312 }
313 // try to add to set of processed files
314 if (workingSet.add(aFile)) {
315 if (logger.isDebugEnabled()) {
316 logger.debug("Scheduling file " + aFile.getName().getPathDecoded() + " for processing");
317 }
318
319 // execute processing in another thread
320 getExecutor().execute(new ExecutorAwareRunnable() {
321 public void run() {
322 String uri = aFile.getName().getURI().toString();
323 Lock lock = lockManager.getLock(uri);
324 if (lock.tryLock()) {
325 processFileNow(aFile);
326 } else {
327 workingSet.remove(aFile);
328 if (logger.isDebugEnabled()) {
329 logger.debug("Unable to acquire lock on " + aFile.getName().getURI());
330 }
331 }
332 }
333 public boolean shouldRunSynchronously(){
334 return !isConcurrentExchange();
335 }
336 });
337 }
338 }
339
340 /**
341 * processes a file
342 *
343 * @param aFile the file to process
344 */
345 protected void processFileNow(FileObject aFile) {
346 try {
347 if (logger.isDebugEnabled()) {
348 logger.debug("Processing file " + aFile.getName().getURI());
349 }
350
351 if (aFile.exists()) {
352 processFile(aFile);
353 }
354 } catch (Exception e) {
355 workingSet.remove(aFile);
356 unlockAsyncFile(aFile);
357 logger.error("Failed to process file: " + aFile.getName().getURI() + ". Reason: " + e, e);
358 }
359 }
360
361 /**
362 * does the real processing logic
363 *
364 * @param file the file to process
365 * @throws Exception on processing errors
366 */
367 protected void processFile(FileObject file) throws Exception {
368 // SM-192: Force close the file, so that the cached informations are cleared
369 file.close();
370
371 String name = file.getName().getURI();
372 FileContent content = file.getContent();
373 content.close();
374
375 InputStream stream = content.getInputStream();
376 if (stream == null) {
377 throw new IOException("No input available for file!");
378 }
379
380 InOnly exchange = getExchangeFactory().createInOnlyExchange();
381 configureExchangeTarget(exchange);
382 NormalizedMessage message = exchange.createMessage();
383 exchange.setInMessage(message);
384 marshaler.readMessage(exchange, message, stream, name);
385
386 // sending the file itself along as a message property and holding on to
387 // the stream we opened
388 exchange.getInMessage().setProperty(VFSComponent.VFS_PROPERTY, file);
389 this.openExchanges.put(exchange.getExchangeId(), stream);
390
391 if(isConcurrentExchange()){
392 send(exchange);
393 }else{
394 sendSync(exchange);
395 process(exchange);
396 }
397 }
398
399 /**
400 * checks if a file is available
401 *
402 * @param aFile the file to check
403 * @return true if available
404 */
405 private boolean isFullyAvailable(FileObject aFile) {
406 try {
407 if (aFile.getContent() != null) {
408 long size_old = aFile.getContent().getSize();
409 try {
410 Thread.sleep(100);
411 } catch (InterruptedException e) {
412 // ignore
413 }
414 long size_new = aFile.getContent().getSize();
415 return (size_old == size_new);
416 }
417 } catch (Exception ex) {
418 // ignore
419 }
420 // default to true
421 return true;
422 }
423
424 /**
425 * Specifies if files should be deleted after they are processed. Default
426 * value is <code>true</code>.
427 *
428 * @param deleteFile a boolean specifying if the file should be deleted
429 */
430 public void setDeleteFile(boolean deleteFile) {
431 this.deleteFile = deleteFile;
432 }
433
434 public boolean isDeleteFile() {
435 return deleteFile;
436 }
437
438 /**
439 * Bean defining the class implementing the file locking strategy. This bean
440 * must be an implementation of the
441 * <code>org.apache.servicemix.locks.LockManager</code> interface. By
442 * default, this will be set to an instances of
443 * <code>org.apache.servicemix.common.locks.impl.SimpleLockManager</code>.
444 *
445 * @param lockManager the <code>LockManager</code> implementation to use
446 */
447 public void setLockManager(LockManager lockManager) {
448 this.lockManager = lockManager;
449 }
450
451 public LockManager getLockManager() {
452 return lockManager;
453 }
454
455 /**
456 * Specifies a <code>FileMarshaler</code> object that will marshal file data
457 * into the NMR. The default file marshaller can read valid XML data.
458 * <code>FileMarshaler</code> objects are implementations of
459 * <code>org.apache.servicemix.components.util.FileMarshaler</code>.
460 *
461 * @param marshaler a <code>FileMarshaler</code> object that can read data
462 * from the file system.
463 */
464 public void setMarshaler(FileMarshaler marshaler) {
465 this.marshaler = marshaler;
466 }
467
468 public FileMarshaler getMarshaler() {
469 return marshaler;
470 }
471
472 /**
473 * Specifies a <code>Comparator</code> object.
474 *
475 * @param comparator a <code>Comparator</code> object.
476 */
477 public void setComparator(Comparator<FileObject> comparator) {
478 this.comparator = comparator;
479 }
480
481 public Comparator<FileObject> getComparator() {
482 return comparator;
483 }
484
485 /**
486 * Specifies a <code>FileSelector</code> object.
487 *
488 * @param selector a <code>FileSelector</code> object
489 */
490 public void setSelector(FileSelector selector) {
491 this.selector = selector;
492 }
493
494 public FileSelector getSelector() {
495 return selector;
496 }
497
498 /**
499 * Specifies a <code>String</code> object representing the path of the
500 * file/folder to be polled.<br /><br />
501 * <b><u>Examples:</u></b><br />
502 * <ul>
503 * <li>file:///home/lhein/pollFolder</li>
504 * <li>zip:file:///home/lhein/pollFolder/myFile.zip</li>
505 * <li>jar:http://www.myhost.com/files/Examples.jar</li>
506 * <li>jar:../lib/classes.jar!/META-INF/manifest.mf</li>
507 * <li>tar:gz:http://anyhost/dir/mytar.tar.gz!/mytar.tar!/path/in/tar/README.txt</li>
508 * <li>tgz:file://anyhost/dir/mytar.tgz!/somepath/somefile</li>
509 * <li>gz:/my/gz/file.gz</li>
510 * <li>http://myusername@somehost/index.html</li>
511 * <li>webdav://somehost:8080/dist</li>
512 * <li>ftp://myusername:mypassword@somehost/pub/downloads/somefile.tgz</li>
513 * <li>sftp://myusername:mypassword@somehost/pub/downloads/somefile.tgz</li>
514 * <li>smb://somehost/home</li>
515 * <li>tmp://dir/somefile.txt</li>
516 * <li>res:path/in/classpath/image.png</li>
517 * <li>ram:///any/path/to/file.txt</li>
518 * <li>mime:file:///your/path/mail/anymail.mime!/filename.pdf</li>
519 * </ul>
520 *
521 * For further details have a look at {@link http://commons.apache.org/vfs/filesystems.html}.
522 * <br /><br />
523 *
524 * @param path a <code>String</code> object that represents a file/folder/vfs
525 */
526 public void setPath(String path) {
527 this.path = path;
528 }
529
530 public String getPath() {
531 return this.path;
532 }
533
534 /**
535 * sets the file system manager
536 *
537 * @param fileSystemManager the file system manager
538 */
539 public void setFileSystemManager(FileSystemManager fileSystemManager) {
540 this.fileSystemManager = fileSystemManager;
541 }
542
543 public FileSystemManager getFileSystemManager() {
544 return this.fileSystemManager;
545 }
546
547 /**
548 * The set of FTPFiles that this component is currently working on
549 *
550 * @return a set of in-process file objects
551 */
552 public Set<FileObject> getWorkingSet() {
553 return workingSet;
554 }
555
556 /**
557 * @return Returns the recursive.
558 */
559 public boolean isRecursive() {
560 return this.recursive;
561 }
562
563 /**
564 * @param recursive The recursive to set.
565 */
566 public void setRecursive(boolean recursive) {
567 this.recursive = recursive;
568 }
569
570 /**
571 * @return the concurrentExchange
572 */
573 public boolean isConcurrentExchange() {
574 return concurrentExchange;
575 }
576
577 /**
578 * @param concurrentExchange the concurrentExchange to set
579 */
580 public void setConcurrentExchange(boolean concurrentExchange) {
581 this.concurrentExchange = concurrentExchange;
582 }
583
584 }