Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.store.kahadb;
018
019import java.io.DataInputStream;
020import java.io.IOException;
021import java.io.InterruptedIOException;
022import java.util.ArrayList;
023import java.util.HashMap;
024import java.util.HashSet;
025import java.util.Iterator;
026import java.util.LinkedList;
027import java.util.List;
028import java.util.Map;
029import java.util.Map.Entry;
030import java.util.Set;
031import java.util.concurrent.BlockingQueue;
032import java.util.concurrent.ExecutorService;
033import java.util.concurrent.FutureTask;
034import java.util.concurrent.LinkedBlockingQueue;
035import java.util.concurrent.Semaphore;
036import java.util.concurrent.ThreadFactory;
037import java.util.concurrent.ThreadPoolExecutor;
038import java.util.concurrent.TimeUnit;
039import java.util.concurrent.TimeoutException;
040import java.util.concurrent.atomic.AtomicBoolean;
041import java.util.concurrent.atomic.AtomicInteger;
042
043import org.apache.activemq.broker.ConnectionContext;
044import org.apache.activemq.broker.region.BaseDestination;
045import org.apache.activemq.broker.scheduler.JobSchedulerStore;
046import org.apache.activemq.command.ActiveMQDestination;
047import org.apache.activemq.command.ActiveMQQueue;
048import org.apache.activemq.command.ActiveMQTempQueue;
049import org.apache.activemq.command.ActiveMQTempTopic;
050import org.apache.activemq.command.ActiveMQTopic;
051import org.apache.activemq.command.Message;
052import org.apache.activemq.command.MessageAck;
053import org.apache.activemq.command.MessageId;
054import org.apache.activemq.command.ProducerId;
055import org.apache.activemq.command.SubscriptionInfo;
056import org.apache.activemq.command.TransactionId;
057import org.apache.activemq.openwire.OpenWireFormat;
058import org.apache.activemq.protobuf.Buffer;
059import org.apache.activemq.store.AbstractMessageStore;
060import org.apache.activemq.store.IndexListener;
061import org.apache.activemq.store.ListenableFuture;
062import org.apache.activemq.store.MessageRecoveryListener;
063import org.apache.activemq.store.MessageStore;
064import org.apache.activemq.store.PersistenceAdapter;
065import org.apache.activemq.store.TopicMessageStore;
066import org.apache.activemq.store.TransactionIdTransformer;
067import org.apache.activemq.store.TransactionStore;
068import org.apache.activemq.store.kahadb.data.KahaAddMessageCommand;
069import org.apache.activemq.store.kahadb.data.KahaDestination;
070import org.apache.activemq.store.kahadb.data.KahaDestination.DestinationType;
071import org.apache.activemq.store.kahadb.data.KahaLocation;
072import org.apache.activemq.store.kahadb.data.KahaRemoveDestinationCommand;
073import org.apache.activemq.store.kahadb.data.KahaRemoveMessageCommand;
074import org.apache.activemq.store.kahadb.data.KahaSubscriptionCommand;
075import org.apache.activemq.store.kahadb.data.KahaUpdateMessageCommand;
076import org.apache.activemq.store.kahadb.disk.journal.Location;
077import org.apache.activemq.store.kahadb.disk.page.Transaction;
078import org.apache.activemq.store.kahadb.scheduler.JobSchedulerStoreImpl;
079import org.apache.activemq.usage.MemoryUsage;
080import org.apache.activemq.usage.SystemUsage;
081import org.apache.activemq.util.ServiceStopper;
082import org.apache.activemq.util.ThreadPoolUtils;
083import org.apache.activemq.wireformat.WireFormat;
084import org.slf4j.Logger;
085import org.slf4j.LoggerFactory;
086
087public class KahaDBStore extends MessageDatabase implements PersistenceAdapter {
088    static final Logger LOG = LoggerFactory.getLogger(KahaDBStore.class);
089    private static final int MAX_ASYNC_JOBS = BaseDestination.MAX_AUDIT_DEPTH;
090
091    public static final String PROPERTY_CANCELED_TASK_MOD_METRIC = "org.apache.activemq.store.kahadb.CANCELED_TASK_MOD_METRIC";
092    public static final int cancelledTaskModMetric = Integer.parseInt(System.getProperty(
093            PROPERTY_CANCELED_TASK_MOD_METRIC, "0"), 10);
094    public static final String PROPERTY_ASYNC_EXECUTOR_MAX_THREADS = "org.apache.activemq.store.kahadb.ASYNC_EXECUTOR_MAX_THREADS";
095    private static final int asyncExecutorMaxThreads = Integer.parseInt(System.getProperty(
096            PROPERTY_ASYNC_EXECUTOR_MAX_THREADS, "1"), 10);;
097
098    protected ExecutorService queueExecutor;
099    protected ExecutorService topicExecutor;
100    protected final List<Map<AsyncJobKey, StoreTask>> asyncQueueMaps = new LinkedList<Map<AsyncJobKey, StoreTask>>();
101    protected final List<Map<AsyncJobKey, StoreTask>> asyncTopicMaps = new LinkedList<Map<AsyncJobKey, StoreTask>>();
102    final WireFormat wireFormat = new OpenWireFormat();
103    private SystemUsage usageManager;
104    private LinkedBlockingQueue<Runnable> asyncQueueJobQueue;
105    private LinkedBlockingQueue<Runnable> asyncTopicJobQueue;
106    Semaphore globalQueueSemaphore;
107    Semaphore globalTopicSemaphore;
108    private boolean concurrentStoreAndDispatchQueues = true;
109    // when true, message order may be compromised when cache is exhausted if store is out
110    // or order w.r.t cache
111    private boolean concurrentStoreAndDispatchTopics = false;
112    private final boolean concurrentStoreAndDispatchTransactions = false;
113    private int maxAsyncJobs = MAX_ASYNC_JOBS;
114    private final KahaDBTransactionStore transactionStore;
115    private TransactionIdTransformer transactionIdTransformer;
116
117    public KahaDBStore() {
118        this.transactionStore = new KahaDBTransactionStore(this);
119        this.transactionIdTransformer = new TransactionIdTransformer() {
120            @Override
121            public TransactionId transform(TransactionId txid) {
122                return txid;
123            }
124        };
125    }
126
127    @Override
128    public String toString() {
129        return "KahaDB:[" + directory.getAbsolutePath() + "]";
130    }
131
132    @Override
133    public void setBrokerName(String brokerName) {
134    }
135
136    @Override
137    public void setUsageManager(SystemUsage usageManager) {
138        this.usageManager = usageManager;
139    }
140
141    public SystemUsage getUsageManager() {
142        return this.usageManager;
143    }
144
145    /**
146     * @return the concurrentStoreAndDispatch
147     */
148    public boolean isConcurrentStoreAndDispatchQueues() {
149        return this.concurrentStoreAndDispatchQueues;
150    }
151
152    /**
153     * @param concurrentStoreAndDispatch
154     *            the concurrentStoreAndDispatch to set
155     */
156    public void setConcurrentStoreAndDispatchQueues(boolean concurrentStoreAndDispatch) {
157        this.concurrentStoreAndDispatchQueues = concurrentStoreAndDispatch;
158    }
159
160    /**
161     * @return the concurrentStoreAndDispatch
162     */
163    public boolean isConcurrentStoreAndDispatchTopics() {
164        return this.concurrentStoreAndDispatchTopics;
165    }
166
167    /**
168     * @param concurrentStoreAndDispatch
169     *            the concurrentStoreAndDispatch to set
170     */
171    public void setConcurrentStoreAndDispatchTopics(boolean concurrentStoreAndDispatch) {
172        this.concurrentStoreAndDispatchTopics = concurrentStoreAndDispatch;
173    }
174
175    public boolean isConcurrentStoreAndDispatchTransactions() {
176        return this.concurrentStoreAndDispatchTransactions;
177    }
178
179    /**
180     * @return the maxAsyncJobs
181     */
182    public int getMaxAsyncJobs() {
183        return this.maxAsyncJobs;
184    }
185
186    /**
187     * @param maxAsyncJobs
188     *            the maxAsyncJobs to set
189     */
190    public void setMaxAsyncJobs(int maxAsyncJobs) {
191        this.maxAsyncJobs = maxAsyncJobs;
192    }
193
194    @Override
195    public void doStart() throws Exception {
196        if (brokerService != null) {
197            metadata.openwireVersion = brokerService.getStoreOpenWireVersion();
198            wireFormat.setVersion(metadata.openwireVersion);
199
200            if (LOG.isDebugEnabled()) {
201                LOG.debug("Store OpenWire version configured as: {}", metadata.openwireVersion);
202            }
203
204        }
205        super.doStart();
206
207        if (brokerService != null) {
208            // In case the recovered store used a different OpenWire version log a warning
209            // to assist in determining why journal reads fail.
210            if (metadata.openwireVersion != brokerService.getStoreOpenWireVersion()) {
211                LOG.warn("Recovered Store uses a different OpenWire version[{}] " +
212                         "than the version configured[{}].",
213                         metadata.openwireVersion, brokerService.getStoreOpenWireVersion());
214            }
215        }
216
217        this.globalQueueSemaphore = new Semaphore(getMaxAsyncJobs());
218        this.globalTopicSemaphore = new Semaphore(getMaxAsyncJobs());
219        this.asyncQueueJobQueue = new LinkedBlockingQueue<Runnable>(getMaxAsyncJobs());
220        this.asyncTopicJobQueue = new LinkedBlockingQueue<Runnable>(getMaxAsyncJobs());
221        this.queueExecutor = new StoreTaskExecutor(1, asyncExecutorMaxThreads, 0L, TimeUnit.MILLISECONDS,
222            asyncQueueJobQueue, new ThreadFactory() {
223                @Override
224                public Thread newThread(Runnable runnable) {
225                    Thread thread = new Thread(runnable, "ConcurrentQueueStoreAndDispatch");
226                    thread.setDaemon(true);
227                    return thread;
228                }
229            });
230        this.topicExecutor = new StoreTaskExecutor(1, asyncExecutorMaxThreads, 0L, TimeUnit.MILLISECONDS,
231            asyncTopicJobQueue, new ThreadFactory() {
232                @Override
233                public Thread newThread(Runnable runnable) {
234                    Thread thread = new Thread(runnable, "ConcurrentTopicStoreAndDispatch");
235                    thread.setDaemon(true);
236                    return thread;
237                }
238            });
239    }
240
241    @Override
242    public void doStop(ServiceStopper stopper) throws Exception {
243        // drain down async jobs
244        LOG.info("Stopping async queue tasks");
245        if (this.globalQueueSemaphore != null) {
246            this.globalQueueSemaphore.tryAcquire(this.maxAsyncJobs, 60, TimeUnit.SECONDS);
247        }
248        synchronized (this.asyncQueueMaps) {
249            for (Map<AsyncJobKey, StoreTask> m : asyncQueueMaps) {
250                synchronized (m) {
251                    for (StoreTask task : m.values()) {
252                        task.cancel();
253                    }
254                }
255            }
256            this.asyncQueueMaps.clear();
257        }
258        LOG.info("Stopping async topic tasks");
259        if (this.globalTopicSemaphore != null) {
260            this.globalTopicSemaphore.tryAcquire(this.maxAsyncJobs, 60, TimeUnit.SECONDS);
261        }
262        synchronized (this.asyncTopicMaps) {
263            for (Map<AsyncJobKey, StoreTask> m : asyncTopicMaps) {
264                synchronized (m) {
265                    for (StoreTask task : m.values()) {
266                        task.cancel();
267                    }
268                }
269            }
270            this.asyncTopicMaps.clear();
271        }
272        if (this.globalQueueSemaphore != null) {
273            this.globalQueueSemaphore.drainPermits();
274        }
275        if (this.globalTopicSemaphore != null) {
276            this.globalTopicSemaphore.drainPermits();
277        }
278        if (this.queueExecutor != null) {
279            ThreadPoolUtils.shutdownNow(queueExecutor);
280            queueExecutor = null;
281        }
282        if (this.topicExecutor != null) {
283            ThreadPoolUtils.shutdownNow(topicExecutor);
284            topicExecutor = null;
285        }
286        LOG.info("Stopped KahaDB");
287        super.doStop(stopper);
288    }
289
290    private Location findMessageLocation(final String key, final KahaDestination destination) throws IOException {
291        return pageFile.tx().execute(new Transaction.CallableClosure<Location, IOException>() {
292            @Override
293            public Location execute(Transaction tx) throws IOException {
294                StoredDestination sd = getStoredDestination(destination, tx);
295                Long sequence = sd.messageIdIndex.get(tx, key);
296                if (sequence == null) {
297                    return null;
298                }
299                return sd.orderIndex.get(tx, sequence).location;
300            }
301        });
302    }
303
304    protected StoreQueueTask removeQueueTask(KahaDBMessageStore store, MessageId id) {
305        StoreQueueTask task = null;
306        synchronized (store.asyncTaskMap) {
307            task = (StoreQueueTask) store.asyncTaskMap.remove(new AsyncJobKey(id, store.getDestination()));
308        }
309        return task;
310    }
311
312    // with asyncTaskMap locked
313    protected void addQueueTask(KahaDBMessageStore store, StoreQueueTask task) throws IOException {
314        store.asyncTaskMap.put(new AsyncJobKey(task.getMessage().getMessageId(), store.getDestination()), task);
315        this.queueExecutor.execute(task);
316    }
317
318    protected StoreTopicTask removeTopicTask(KahaDBTopicMessageStore store, MessageId id) {
319        StoreTopicTask task = null;
320        synchronized (store.asyncTaskMap) {
321            task = (StoreTopicTask) store.asyncTaskMap.remove(new AsyncJobKey(id, store.getDestination()));
322        }
323        return task;
324    }
325
326    protected void addTopicTask(KahaDBTopicMessageStore store, StoreTopicTask task) throws IOException {
327        synchronized (store.asyncTaskMap) {
328            store.asyncTaskMap.put(new AsyncJobKey(task.getMessage().getMessageId(), store.getDestination()), task);
329        }
330        this.topicExecutor.execute(task);
331    }
332
333    @Override
334    public TransactionStore createTransactionStore() throws IOException {
335        return this.transactionStore;
336    }
337
338    public boolean getForceRecoverIndex() {
339        return this.forceRecoverIndex;
340    }
341
342    public void setForceRecoverIndex(boolean forceRecoverIndex) {
343        this.forceRecoverIndex = forceRecoverIndex;
344    }
345
346    public class KahaDBMessageStore extends AbstractMessageStore {
347        protected final Map<AsyncJobKey, StoreTask> asyncTaskMap = new HashMap<AsyncJobKey, StoreTask>();
348        protected KahaDestination dest;
349        private final int maxAsyncJobs;
350        private final Semaphore localDestinationSemaphore;
351
352        double doneTasks, canceledTasks = 0;
353
354        public KahaDBMessageStore(ActiveMQDestination destination) {
355            super(destination);
356            this.dest = convert(destination);
357            this.maxAsyncJobs = getMaxAsyncJobs();
358            this.localDestinationSemaphore = new Semaphore(this.maxAsyncJobs);
359        }
360
361        @Override
362        public ActiveMQDestination getDestination() {
363            return destination;
364        }
365
366        @Override
367        public ListenableFuture<Object> asyncAddQueueMessage(final ConnectionContext context, final Message message)
368                throws IOException {
369            if (isConcurrentStoreAndDispatchQueues()) {
370                StoreQueueTask result = new StoreQueueTask(this, context, message);
371                ListenableFuture<Object> future = result.getFuture();
372                message.getMessageId().setFutureOrSequenceLong(future);
373                message.setRecievedByDFBridge(true); // flag message as concurrentStoreAndDispatch
374                result.aquireLocks();
375                synchronized (asyncTaskMap) {
376                    addQueueTask(this, result);
377                    if (indexListener != null) {
378                        indexListener.onAdd(new IndexListener.MessageContext(context, message, null));
379                    }
380                }
381                return future;
382            } else {
383                return super.asyncAddQueueMessage(context, message);
384            }
385        }
386
387        @Override
388        public void removeAsyncMessage(ConnectionContext context, MessageAck ack) throws IOException {
389            if (isConcurrentStoreAndDispatchQueues()) {
390                AsyncJobKey key = new AsyncJobKey(ack.getLastMessageId(), getDestination());
391                StoreQueueTask task = null;
392                synchronized (asyncTaskMap) {
393                    task = (StoreQueueTask) asyncTaskMap.get(key);
394                }
395                if (task != null) {
396                    if (ack.isInTransaction() || !task.cancel()) {
397                        try {
398                            task.future.get();
399                        } catch (InterruptedException e) {
400                            throw new InterruptedIOException(e.toString());
401                        } catch (Exception ignored) {
402                            LOG.debug("removeAsync: cannot cancel, waiting for add resulted in ex", ignored);
403                        }
404                        removeMessage(context, ack);
405                    } else {
406                        indexLock.writeLock().lock();
407                        try {
408                            metadata.producerSequenceIdTracker.isDuplicate(ack.getLastMessageId());
409                        } finally {
410                            indexLock.writeLock().unlock();
411                        }
412                        synchronized (asyncTaskMap) {
413                            asyncTaskMap.remove(key);
414                        }
415                    }
416                } else {
417                    removeMessage(context, ack);
418                }
419            } else {
420                removeMessage(context, ack);
421            }
422        }
423
424        @Override
425        public void addMessage(final ConnectionContext context, final Message message) throws IOException {
426            final KahaAddMessageCommand command = new KahaAddMessageCommand();
427            command.setDestination(dest);
428            command.setMessageId(message.getMessageId().toProducerKey());
429            command.setTransactionInfo(TransactionIdConversion.convert(transactionIdTransformer.transform(message.getTransactionId())));
430            command.setPriority(message.getPriority());
431            command.setPrioritySupported(isPrioritizedMessages());
432            org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(message);
433            command.setMessage(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
434            store(command, isEnableJournalDiskSyncs() && message.isResponseRequired(), new IndexAware() {
435                // sync add? (for async, future present from getFutureOrSequenceLong)
436                Object possibleFuture = message.getMessageId().getFutureOrSequenceLong();
437
438                @Override
439                public void sequenceAssignedWithIndexLocked(final long sequence) {
440                    message.getMessageId().setFutureOrSequenceLong(sequence);
441                    if (indexListener != null) {
442                        if (possibleFuture == null) {
443                            trackPendingAdd(dest, sequence);
444                            indexListener.onAdd(new IndexListener.MessageContext(context, message, new Runnable() {
445                                @Override
446                                public void run() {
447                                    trackPendingAddComplete(dest, sequence);
448                                }
449                            }));
450                        }
451                    }
452                }
453            }, null);
454        }
455
456        @Override
457        public void updateMessage(Message message) throws IOException {
458            if (LOG.isTraceEnabled()) {
459                LOG.trace("updating: " + message.getMessageId() + " with deliveryCount: " + message.getRedeliveryCounter());
460            }
461            KahaUpdateMessageCommand updateMessageCommand = new KahaUpdateMessageCommand();
462            KahaAddMessageCommand command = new KahaAddMessageCommand();
463            command.setDestination(dest);
464            command.setMessageId(message.getMessageId().toProducerKey());
465            command.setPriority(message.getPriority());
466            command.setPrioritySupported(prioritizedMessages);
467            org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(message);
468            command.setMessage(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
469            updateMessageCommand.setMessage(command);
470            store(updateMessageCommand, isEnableJournalDiskSyncs(), null, null);
471        }
472
473        @Override
474        public void removeMessage(ConnectionContext context, MessageAck ack) throws IOException {
475            KahaRemoveMessageCommand command = new KahaRemoveMessageCommand();
476            command.setDestination(dest);
477            command.setMessageId(ack.getLastMessageId().toProducerKey());
478            command.setTransactionInfo(TransactionIdConversion.convert(transactionIdTransformer.transform(ack.getTransactionId())));
479
480            org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(ack);
481            command.setAck(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
482            store(command, isEnableJournalDiskSyncs() && ack.isResponseRequired(), null, null);
483        }
484
485        @Override
486        public void removeAllMessages(ConnectionContext context) throws IOException {
487            KahaRemoveDestinationCommand command = new KahaRemoveDestinationCommand();
488            command.setDestination(dest);
489            store(command, true, null, null);
490        }
491
492        @Override
493        public Message getMessage(MessageId identity) throws IOException {
494            final String key = identity.toProducerKey();
495
496            // Hopefully one day the page file supports concurrent read
497            // operations... but for now we must
498            // externally synchronize...
499            Location location;
500            indexLock.writeLock().lock();
501            try {
502                location = findMessageLocation(key, dest);
503            } finally {
504                indexLock.writeLock().unlock();
505            }
506            if (location == null) {
507                return null;
508            }
509
510            return loadMessage(location);
511        }
512
513        @Override
514        public int getMessageCount() throws IOException {
515            try {
516                lockAsyncJobQueue();
517                indexLock.writeLock().lock();
518                try {
519                    return pageFile.tx().execute(new Transaction.CallableClosure<Integer, IOException>() {
520                        @Override
521                        public Integer execute(Transaction tx) throws IOException {
522                            // Iterate through all index entries to get a count
523                            // of messages in the destination.
524                            StoredDestination sd = getStoredDestination(dest, tx);
525                            int rc = 0;
526                            for (Iterator<Entry<Location, Long>> iterator = sd.locationIndex.iterator(tx); iterator.hasNext();) {
527                                iterator.next();
528                                rc++;
529                            }
530                            return rc;
531                        }
532                    });
533                } finally {
534                    indexLock.writeLock().unlock();
535                }
536            } finally {
537                unlockAsyncJobQueue();
538            }
539        }
540
541        @Override
542        public boolean isEmpty() throws IOException {
543            indexLock.writeLock().lock();
544            try {
545                return pageFile.tx().execute(new Transaction.CallableClosure<Boolean, IOException>() {
546                    @Override
547                    public Boolean execute(Transaction tx) throws IOException {
548                        // Iterate through all index entries to get a count of
549                        // messages in the destination.
550                        StoredDestination sd = getStoredDestination(dest, tx);
551                        return sd.locationIndex.isEmpty(tx);
552                    }
553                });
554            } finally {
555                indexLock.writeLock().unlock();
556            }
557        }
558
559        @Override
560        public void recover(final MessageRecoveryListener listener) throws Exception {
561            // recovery may involve expiry which will modify
562            indexLock.writeLock().lock();
563            try {
564                pageFile.tx().execute(new Transaction.Closure<Exception>() {
565                    @Override
566                    public void execute(Transaction tx) throws Exception {
567                        StoredDestination sd = getStoredDestination(dest, tx);
568                        recoverRolledBackAcks(sd, tx, Integer.MAX_VALUE, listener);
569                        sd.orderIndex.resetCursorPosition();
570                        for (Iterator<Entry<Long, MessageKeys>> iterator = sd.orderIndex.iterator(tx); listener.hasSpace() && iterator
571                                .hasNext(); ) {
572                            Entry<Long, MessageKeys> entry = iterator.next();
573                            if (ackedAndPrepared.contains(entry.getValue().messageId)) {
574                                continue;
575                            }
576                            Message msg = loadMessage(entry.getValue().location);
577                            listener.recoverMessage(msg);
578                        }
579                    }
580                });
581            } finally {
582                indexLock.writeLock().unlock();
583            }
584        }
585
586        @Override
587        public void recoverNextMessages(final int maxReturned, final MessageRecoveryListener listener) throws Exception {
588            indexLock.writeLock().lock();
589            try {
590                pageFile.tx().execute(new Transaction.Closure<Exception>() {
591                    @Override
592                    public void execute(Transaction tx) throws Exception {
593                        StoredDestination sd = getStoredDestination(dest, tx);
594                        Entry<Long, MessageKeys> entry = null;
595                        int counter = recoverRolledBackAcks(sd, tx, maxReturned, listener);
596                        for (Iterator<Entry<Long, MessageKeys>> iterator = sd.orderIndex.iterator(tx); iterator.hasNext(); ) {
597                            entry = iterator.next();
598                            if (ackedAndPrepared.contains(entry.getValue().messageId)) {
599                                continue;
600                            }
601                            Message msg = loadMessage(entry.getValue().location);
602                            msg.getMessageId().setFutureOrSequenceLong(entry.getKey());
603                            listener.recoverMessage(msg);
604                            counter++;
605                            if (counter >= maxReturned) {
606                                break;
607                            }
608                        }
609                        sd.orderIndex.stoppedIterating();
610                    }
611                });
612            } finally {
613                indexLock.writeLock().unlock();
614            }
615        }
616
617        protected int recoverRolledBackAcks(StoredDestination sd, Transaction tx, int maxReturned, MessageRecoveryListener listener) throws Exception {
618            int counter = 0;
619            String id;
620            for (Iterator<String> iterator = rolledBackAcks.iterator(); iterator.hasNext(); ) {
621                id = iterator.next();
622                iterator.remove();
623                Long sequence = sd.messageIdIndex.get(tx, id);
624                if (sequence != null) {
625                    if (sd.orderIndex.alreadyDispatched(sequence)) {
626                        listener.recoverMessage(loadMessage(sd.orderIndex.get(tx, sequence).location));
627                        counter++;
628                        if (counter >= maxReturned) {
629                            break;
630                        }
631                    } else {
632                        LOG.info("rolledback ack message {} with seq {} will be picked up in future batch {}", id, sequence, sd.orderIndex.cursor);
633                    }
634                } else {
635                    LOG.warn("Failed to locate rolled back ack message {} in {}", id, sd);
636                }
637            }
638            return counter;
639        }
640
641
642        @Override
643        public void resetBatching() {
644            if (pageFile.isLoaded()) {
645                indexLock.writeLock().lock();
646                try {
647                    pageFile.tx().execute(new Transaction.Closure<Exception>() {
648                        @Override
649                        public void execute(Transaction tx) throws Exception {
650                            StoredDestination sd = getExistingStoredDestination(dest, tx);
651                            if (sd != null) {
652                                sd.orderIndex.resetCursorPosition();}
653                            }
654                        });
655                } catch (Exception e) {
656                    LOG.error("Failed to reset batching",e);
657                } finally {
658                    indexLock.writeLock().unlock();
659                }
660            }
661        }
662
663        @Override
664        public void setBatch(final MessageId identity) throws IOException {
665            indexLock.writeLock().lock();
666            try {
667                pageFile.tx().execute(new Transaction.Closure<IOException>() {
668                    @Override
669                    public void execute(Transaction tx) throws IOException {
670                        StoredDestination sd = getStoredDestination(dest, tx);
671                        Long location = (Long) identity.getFutureOrSequenceLong();
672                        Long pending = sd.orderIndex.minPendingAdd();
673                        if (pending != null) {
674                            location = Math.min(location, pending-1);
675                        }
676                        sd.orderIndex.setBatch(tx, location);
677                    }
678                });
679            } finally {
680                indexLock.writeLock().unlock();
681            }
682        }
683
684        @Override
685        public void setMemoryUsage(MemoryUsage memoryUsage) {
686        }
687        @Override
688        public void start() throws Exception {
689            super.start();
690        }
691        @Override
692        public void stop() throws Exception {
693            super.stop();
694        }
695
696        protected void lockAsyncJobQueue() {
697            try {
698                if (!this.localDestinationSemaphore.tryAcquire(this.maxAsyncJobs, 60, TimeUnit.SECONDS)) {
699                    throw new TimeoutException(this +" timeout waiting for localDestSem:" + this.localDestinationSemaphore);
700                }
701            } catch (Exception e) {
702                LOG.error("Failed to lock async jobs for " + this.destination, e);
703            }
704        }
705
706        protected void unlockAsyncJobQueue() {
707            this.localDestinationSemaphore.release(this.maxAsyncJobs);
708        }
709
710        protected void acquireLocalAsyncLock() {
711            try {
712                this.localDestinationSemaphore.acquire();
713            } catch (InterruptedException e) {
714                LOG.error("Failed to aquire async lock for " + this.destination, e);
715            }
716        }
717
718        protected void releaseLocalAsyncLock() {
719            this.localDestinationSemaphore.release();
720        }
721
722        @Override
723        public String toString(){
724            return "permits:" + this.localDestinationSemaphore.availablePermits() + ",sd=" + storedDestinations.get(key(dest));
725        }
726    }
727
728    class KahaDBTopicMessageStore extends KahaDBMessageStore implements TopicMessageStore {
729        private final AtomicInteger subscriptionCount = new AtomicInteger();
730        public KahaDBTopicMessageStore(ActiveMQTopic destination) throws IOException {
731            super(destination);
732            this.subscriptionCount.set(getAllSubscriptions().length);
733            if (isConcurrentStoreAndDispatchTopics()) {
734                asyncTopicMaps.add(asyncTaskMap);
735            }
736        }
737
738        @Override
739        public ListenableFuture<Object> asyncAddTopicMessage(final ConnectionContext context, final Message message)
740                throws IOException {
741            if (isConcurrentStoreAndDispatchTopics()) {
742                StoreTopicTask result = new StoreTopicTask(this, context, message, subscriptionCount.get());
743                result.aquireLocks();
744                addTopicTask(this, result);
745                return result.getFuture();
746            } else {
747                return super.asyncAddTopicMessage(context, message);
748            }
749        }
750
751        @Override
752        public void acknowledge(ConnectionContext context, String clientId, String subscriptionName,
753                                MessageId messageId, MessageAck ack) throws IOException {
754            String subscriptionKey = subscriptionKey(clientId, subscriptionName).toString();
755            if (isConcurrentStoreAndDispatchTopics()) {
756                AsyncJobKey key = new AsyncJobKey(messageId, getDestination());
757                StoreTopicTask task = null;
758                synchronized (asyncTaskMap) {
759                    task = (StoreTopicTask) asyncTaskMap.get(key);
760                }
761                if (task != null) {
762                    if (task.addSubscriptionKey(subscriptionKey)) {
763                        removeTopicTask(this, messageId);
764                        if (task.cancel()) {
765                            synchronized (asyncTaskMap) {
766                                asyncTaskMap.remove(key);
767                            }
768                        }
769                    }
770                } else {
771                    doAcknowledge(context, subscriptionKey, messageId, ack);
772                }
773            } else {
774                doAcknowledge(context, subscriptionKey, messageId, ack);
775            }
776        }
777
778        protected void doAcknowledge(ConnectionContext context, String subscriptionKey, MessageId messageId, MessageAck ack)
779                throws IOException {
780            KahaRemoveMessageCommand command = new KahaRemoveMessageCommand();
781            command.setDestination(dest);
782            command.setSubscriptionKey(subscriptionKey);
783            command.setMessageId(messageId.toProducerKey());
784            command.setTransactionInfo(ack != null ? TransactionIdConversion.convert(transactionIdTransformer.transform(ack.getTransactionId())) : null);
785            if (ack != null && ack.isUnmatchedAck()) {
786                command.setAck(UNMATCHED);
787            } else {
788                org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(ack);
789                command.setAck(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
790            }
791            store(command, false, null, null);
792        }
793
794        @Override
795        public void addSubscription(SubscriptionInfo subscriptionInfo, boolean retroactive) throws IOException {
796            String subscriptionKey = subscriptionKey(subscriptionInfo.getClientId(), subscriptionInfo
797                    .getSubscriptionName());
798            KahaSubscriptionCommand command = new KahaSubscriptionCommand();
799            command.setDestination(dest);
800            command.setSubscriptionKey(subscriptionKey.toString());
801            command.setRetroactive(retroactive);
802            org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(subscriptionInfo);
803            command.setSubscriptionInfo(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
804            store(command, isEnableJournalDiskSyncs() && true, null, null);
805            this.subscriptionCount.incrementAndGet();
806        }
807
808        @Override
809        public void deleteSubscription(String clientId, String subscriptionName) throws IOException {
810            KahaSubscriptionCommand command = new KahaSubscriptionCommand();
811            command.setDestination(dest);
812            command.setSubscriptionKey(subscriptionKey(clientId, subscriptionName).toString());
813            store(command, isEnableJournalDiskSyncs() && true, null, null);
814            this.subscriptionCount.decrementAndGet();
815        }
816
817        @Override
818        public SubscriptionInfo[] getAllSubscriptions() throws IOException {
819
820            final ArrayList<SubscriptionInfo> subscriptions = new ArrayList<SubscriptionInfo>();
821            indexLock.writeLock().lock();
822            try {
823                pageFile.tx().execute(new Transaction.Closure<IOException>() {
824                    @Override
825                    public void execute(Transaction tx) throws IOException {
826                        StoredDestination sd = getStoredDestination(dest, tx);
827                        for (Iterator<Entry<String, KahaSubscriptionCommand>> iterator = sd.subscriptions.iterator(tx); iterator
828                                .hasNext();) {
829                            Entry<String, KahaSubscriptionCommand> entry = iterator.next();
830                            SubscriptionInfo info = (SubscriptionInfo) wireFormat.unmarshal(new DataInputStream(entry
831                                    .getValue().getSubscriptionInfo().newInput()));
832                            subscriptions.add(info);
833
834                        }
835                    }
836                });
837            } finally {
838                indexLock.writeLock().unlock();
839            }
840
841            SubscriptionInfo[] rc = new SubscriptionInfo[subscriptions.size()];
842            subscriptions.toArray(rc);
843            return rc;
844        }
845
846        @Override
847        public SubscriptionInfo lookupSubscription(String clientId, String subscriptionName) throws IOException {
848            final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
849            indexLock.writeLock().lock();
850            try {
851                return pageFile.tx().execute(new Transaction.CallableClosure<SubscriptionInfo, IOException>() {
852                    @Override
853                    public SubscriptionInfo execute(Transaction tx) throws IOException {
854                        StoredDestination sd = getStoredDestination(dest, tx);
855                        KahaSubscriptionCommand command = sd.subscriptions.get(tx, subscriptionKey);
856                        if (command == null) {
857                            return null;
858                        }
859                        return (SubscriptionInfo) wireFormat.unmarshal(new DataInputStream(command
860                                .getSubscriptionInfo().newInput()));
861                    }
862                });
863            } finally {
864                indexLock.writeLock().unlock();
865            }
866        }
867
868        @Override
869        public int getMessageCount(String clientId, String subscriptionName) throws IOException {
870            final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
871            indexLock.writeLock().lock();
872            try {
873                return pageFile.tx().execute(new Transaction.CallableClosure<Integer, IOException>() {
874                    @Override
875                    public Integer execute(Transaction tx) throws IOException {
876                        StoredDestination sd = getStoredDestination(dest, tx);
877                        LastAck cursorPos = getLastAck(tx, sd, subscriptionKey);
878                        if (cursorPos == null) {
879                            // The subscription might not exist.
880                            return 0;
881                        }
882
883                        return (int) getStoredMessageCount(tx, sd, subscriptionKey);
884                    }
885                });
886            } finally {
887                indexLock.writeLock().unlock();
888            }
889        }
890
891        @Override
892        public void recoverSubscription(String clientId, String subscriptionName, final MessageRecoveryListener listener)
893                throws Exception {
894            final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
895            @SuppressWarnings("unused")
896            final SubscriptionInfo info = lookupSubscription(clientId, subscriptionName);
897            indexLock.writeLock().lock();
898            try {
899                pageFile.tx().execute(new Transaction.Closure<Exception>() {
900                    @Override
901                    public void execute(Transaction tx) throws Exception {
902                        StoredDestination sd = getStoredDestination(dest, tx);
903                        LastAck cursorPos = getLastAck(tx, sd, subscriptionKey);
904                        sd.orderIndex.setBatch(tx, cursorPos);
905                        recoverRolledBackAcks(sd, tx, Integer.MAX_VALUE, listener);
906                        for (Iterator<Entry<Long, MessageKeys>> iterator = sd.orderIndex.iterator(tx); iterator
907                                .hasNext();) {
908                            Entry<Long, MessageKeys> entry = iterator.next();
909                            if (ackedAndPrepared.contains(entry.getValue().messageId)) {
910                                continue;
911                            }
912                            listener.recoverMessage(loadMessage(entry.getValue().location));
913                        }
914                        sd.orderIndex.resetCursorPosition();
915                    }
916                });
917            } finally {
918                indexLock.writeLock().unlock();
919            }
920        }
921
922        @Override
923        public void recoverNextMessages(String clientId, String subscriptionName, final int maxReturned,
924                final MessageRecoveryListener listener) throws Exception {
925            final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
926            @SuppressWarnings("unused")
927            final SubscriptionInfo info = lookupSubscription(clientId, subscriptionName);
928            indexLock.writeLock().lock();
929            try {
930                pageFile.tx().execute(new Transaction.Closure<Exception>() {
931                    @Override
932                    public void execute(Transaction tx) throws Exception {
933                        StoredDestination sd = getStoredDestination(dest, tx);
934                        sd.orderIndex.resetCursorPosition();
935                        MessageOrderCursor moc = sd.subscriptionCursors.get(subscriptionKey);
936                        if (moc == null) {
937                            LastAck pos = getLastAck(tx, sd, subscriptionKey);
938                            if (pos == null) {
939                                // sub deleted
940                                return;
941                            }
942                            sd.orderIndex.setBatch(tx, pos);
943                            moc = sd.orderIndex.cursor;
944                        } else {
945                            sd.orderIndex.cursor.sync(moc);
946                        }
947
948                        Entry<Long, MessageKeys> entry = null;
949                        int counter = recoverRolledBackAcks(sd, tx, maxReturned, listener);
950                        for (Iterator<Entry<Long, MessageKeys>> iterator = sd.orderIndex.iterator(tx, moc); iterator
951                                .hasNext();) {
952                            entry = iterator.next();
953                            if (ackedAndPrepared.contains(entry.getValue().messageId)) {
954                                continue;
955                            }
956                            if (listener.recoverMessage(loadMessage(entry.getValue().location))) {
957                                counter++;
958                            }
959                            if (counter >= maxReturned || listener.hasSpace() == false) {
960                                break;
961                            }
962                        }
963                        sd.orderIndex.stoppedIterating();
964                        if (entry != null) {
965                            MessageOrderCursor copy = sd.orderIndex.cursor.copy();
966                            sd.subscriptionCursors.put(subscriptionKey, copy);
967                        }
968                    }
969                });
970            } finally {
971                indexLock.writeLock().unlock();
972            }
973        }
974
975        @Override
976        public void resetBatching(String clientId, String subscriptionName) {
977            try {
978                final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
979                indexLock.writeLock().lock();
980                try {
981                    pageFile.tx().execute(new Transaction.Closure<IOException>() {
982                        @Override
983                        public void execute(Transaction tx) throws IOException {
984                            StoredDestination sd = getStoredDestination(dest, tx);
985                            sd.subscriptionCursors.remove(subscriptionKey);
986                        }
987                    });
988                }finally {
989                    indexLock.writeLock().unlock();
990                }
991            } catch (IOException e) {
992                throw new RuntimeException(e);
993            }
994        }
995    }
996
997    String subscriptionKey(String clientId, String subscriptionName) {
998        return clientId + ":" + subscriptionName;
999    }
1000
1001    @Override
1002    public MessageStore createQueueMessageStore(ActiveMQQueue destination) throws IOException {
1003        return this.transactionStore.proxy(new KahaDBMessageStore(destination));
1004    }
1005
1006    @Override
1007    public TopicMessageStore createTopicMessageStore(ActiveMQTopic destination) throws IOException {
1008        return this.transactionStore.proxy(new KahaDBTopicMessageStore(destination));
1009    }
1010
1011    /**
1012     * Cleanup method to remove any state associated with the given destination.
1013     * This method does not stop the message store (it might not be cached).
1014     *
1015     * @param destination
1016     *            Destination to forget
1017     */
1018    @Override
1019    public void removeQueueMessageStore(ActiveMQQueue destination) {
1020    }
1021
1022    /**
1023     * Cleanup method to remove any state associated with the given destination
1024     * This method does not stop the message store (it might not be cached).
1025     *
1026     * @param destination
1027     *            Destination to forget
1028     */
1029    @Override
1030    public void removeTopicMessageStore(ActiveMQTopic destination) {
1031    }
1032
1033    @Override
1034    public void deleteAllMessages() throws IOException {
1035        deleteAllMessages = true;
1036    }
1037
1038    @Override
1039    public Set<ActiveMQDestination> getDestinations() {
1040        try {
1041            final HashSet<ActiveMQDestination> rc = new HashSet<ActiveMQDestination>();
1042            indexLock.writeLock().lock();
1043            try {
1044                pageFile.tx().execute(new Transaction.Closure<IOException>() {
1045                    @Override
1046                    public void execute(Transaction tx) throws IOException {
1047                        for (Iterator<Entry<String, StoredDestination>> iterator = metadata.destinations.iterator(tx); iterator
1048                                .hasNext();) {
1049                            Entry<String, StoredDestination> entry = iterator.next();
1050                            if (!isEmptyTopic(entry, tx)) {
1051                                rc.add(convert(entry.getKey()));
1052                            }
1053                        }
1054                    }
1055
1056                    private boolean isEmptyTopic(Entry<String, StoredDestination> entry, Transaction tx)
1057                            throws IOException {
1058                        boolean isEmptyTopic = false;
1059                        ActiveMQDestination dest = convert(entry.getKey());
1060                        if (dest.isTopic()) {
1061                            StoredDestination loadedStore = getStoredDestination(convert(dest), tx);
1062                            if (loadedStore.subscriptionAcks.isEmpty(tx)) {
1063                                isEmptyTopic = true;
1064                            }
1065                        }
1066                        return isEmptyTopic;
1067                    }
1068                });
1069            }finally {
1070                indexLock.writeLock().unlock();
1071            }
1072            return rc;
1073        } catch (IOException e) {
1074            throw new RuntimeException(e);
1075        }
1076    }
1077
1078    @Override
1079    public long getLastMessageBrokerSequenceId() throws IOException {
1080        return 0;
1081    }
1082
1083    @Override
1084    public long getLastProducerSequenceId(ProducerId id) {
1085        indexLock.writeLock().lock();
1086        try {
1087            return metadata.producerSequenceIdTracker.getLastSeqId(id);
1088        } finally {
1089            indexLock.writeLock().unlock();
1090        }
1091    }
1092
1093    @Override
1094    public long size() {
1095        try {
1096            return journalSize.get() + getPageFile().getDiskSize();
1097        } catch (IOException e) {
1098            throw new RuntimeException(e);
1099        }
1100    }
1101
1102    @Override
1103    public void beginTransaction(ConnectionContext context) throws IOException {
1104        throw new IOException("Not yet implemented.");
1105    }
1106    @Override
1107    public void commitTransaction(ConnectionContext context) throws IOException {
1108        throw new IOException("Not yet implemented.");
1109    }
1110    @Override
1111    public void rollbackTransaction(ConnectionContext context) throws IOException {
1112        throw new IOException("Not yet implemented.");
1113    }
1114
1115    @Override
1116    public void checkpoint(boolean sync) throws IOException {
1117        super.checkpointCleanup(sync);
1118    }
1119
1120    // /////////////////////////////////////////////////////////////////
1121    // Internal helper methods.
1122    // /////////////////////////////////////////////////////////////////
1123
1124    /**
1125     * @param location
1126     * @return
1127     * @throws IOException
1128     */
1129    Message loadMessage(Location location) throws IOException {
1130        try {
1131            JournalCommand<?> command = load(location);
1132            KahaAddMessageCommand addMessage = null;
1133            switch (command.type()) {
1134                case KAHA_UPDATE_MESSAGE_COMMAND:
1135                    addMessage = ((KahaUpdateMessageCommand) command).getMessage();
1136                    break;
1137                default:
1138                    addMessage = (KahaAddMessageCommand) command;
1139            }
1140            Message msg = (Message) wireFormat.unmarshal(new DataInputStream(addMessage.getMessage().newInput()));
1141            return msg;
1142        } catch (IOException ioe) {
1143            LOG.error("Failed to load message at: {}", location , ioe);
1144            brokerService.handleIOException(ioe);
1145            throw ioe;
1146        }
1147    }
1148
1149    // /////////////////////////////////////////////////////////////////
1150    // Internal conversion methods.
1151    // /////////////////////////////////////////////////////////////////
1152
1153    KahaLocation convert(Location location) {
1154        KahaLocation rc = new KahaLocation();
1155        rc.setLogId(location.getDataFileId());
1156        rc.setOffset(location.getOffset());
1157        return rc;
1158    }
1159
1160    KahaDestination convert(ActiveMQDestination dest) {
1161        KahaDestination rc = new KahaDestination();
1162        rc.setName(dest.getPhysicalName());
1163        switch (dest.getDestinationType()) {
1164        case ActiveMQDestination.QUEUE_TYPE:
1165            rc.setType(DestinationType.QUEUE);
1166            return rc;
1167        case ActiveMQDestination.TOPIC_TYPE:
1168            rc.setType(DestinationType.TOPIC);
1169            return rc;
1170        case ActiveMQDestination.TEMP_QUEUE_TYPE:
1171            rc.setType(DestinationType.TEMP_QUEUE);
1172            return rc;
1173        case ActiveMQDestination.TEMP_TOPIC_TYPE:
1174            rc.setType(DestinationType.TEMP_TOPIC);
1175            return rc;
1176        default:
1177            return null;
1178        }
1179    }
1180
1181    ActiveMQDestination convert(String dest) {
1182        int p = dest.indexOf(":");
1183        if (p < 0) {
1184            throw new IllegalArgumentException("Not in the valid destination format");
1185        }
1186        int type = Integer.parseInt(dest.substring(0, p));
1187        String name = dest.substring(p + 1);
1188        return convert(type, name);
1189    }
1190
1191    private ActiveMQDestination convert(KahaDestination commandDestination) {
1192        return convert(commandDestination.getType().getNumber(), commandDestination.getName());
1193    }
1194
1195    private ActiveMQDestination convert(int type, String name) {
1196        switch (KahaDestination.DestinationType.valueOf(type)) {
1197        case QUEUE:
1198            return new ActiveMQQueue(name);
1199        case TOPIC:
1200            return new ActiveMQTopic(name);
1201        case TEMP_QUEUE:
1202            return new ActiveMQTempQueue(name);
1203        case TEMP_TOPIC:
1204            return new ActiveMQTempTopic(name);
1205        default:
1206            throw new IllegalArgumentException("Not in the valid destination format");
1207        }
1208    }
1209
1210    public TransactionIdTransformer getTransactionIdTransformer() {
1211        return transactionIdTransformer;
1212    }
1213
1214    public void setTransactionIdTransformer(TransactionIdTransformer transactionIdTransformer) {
1215        this.transactionIdTransformer = transactionIdTransformer;
1216    }
1217
1218    static class AsyncJobKey {
1219        MessageId id;
1220        ActiveMQDestination destination;
1221
1222        AsyncJobKey(MessageId id, ActiveMQDestination destination) {
1223            this.id = id;
1224            this.destination = destination;
1225        }
1226
1227        @Override
1228        public boolean equals(Object obj) {
1229            if (obj == this) {
1230                return true;
1231            }
1232            return obj instanceof AsyncJobKey && id.equals(((AsyncJobKey) obj).id)
1233                    && destination.equals(((AsyncJobKey) obj).destination);
1234        }
1235
1236        @Override
1237        public int hashCode() {
1238            return id.hashCode() + destination.hashCode();
1239        }
1240
1241        @Override
1242        public String toString() {
1243            return destination.getPhysicalName() + "-" + id;
1244        }
1245    }
1246
1247    public interface StoreTask {
1248        public boolean cancel();
1249
1250        public void aquireLocks();
1251
1252        public void releaseLocks();
1253    }
1254
1255    class StoreQueueTask implements Runnable, StoreTask {
1256        protected final Message message;
1257        protected final ConnectionContext context;
1258        protected final KahaDBMessageStore store;
1259        protected final InnerFutureTask future;
1260        protected final AtomicBoolean done = new AtomicBoolean();
1261        protected final AtomicBoolean locked = new AtomicBoolean();
1262
1263        public StoreQueueTask(KahaDBMessageStore store, ConnectionContext context, Message message) {
1264            this.store = store;
1265            this.context = context;
1266            this.message = message;
1267            this.future = new InnerFutureTask(this);
1268        }
1269
1270        public ListenableFuture<Object> getFuture() {
1271            return this.future;
1272        }
1273
1274        @Override
1275        public boolean cancel() {
1276            if (this.done.compareAndSet(false, true)) {
1277                return this.future.cancel(false);
1278            }
1279            return false;
1280        }
1281
1282        @Override
1283        public void aquireLocks() {
1284            if (this.locked.compareAndSet(false, true)) {
1285                try {
1286                    globalQueueSemaphore.acquire();
1287                    store.acquireLocalAsyncLock();
1288                    message.incrementReferenceCount();
1289                } catch (InterruptedException e) {
1290                    LOG.warn("Failed to aquire lock", e);
1291                }
1292            }
1293
1294        }
1295
1296        @Override
1297        public void releaseLocks() {
1298            if (this.locked.compareAndSet(true, false)) {
1299                store.releaseLocalAsyncLock();
1300                globalQueueSemaphore.release();
1301                message.decrementReferenceCount();
1302            }
1303        }
1304
1305        @Override
1306        public void run() {
1307            this.store.doneTasks++;
1308            try {
1309                if (this.done.compareAndSet(false, true)) {
1310                    this.store.addMessage(context, message);
1311                    removeQueueTask(this.store, this.message.getMessageId());
1312                    this.future.complete();
1313                } else if (cancelledTaskModMetric > 0 && this.store.canceledTasks++ % cancelledTaskModMetric == 0) {
1314                    System.err.println(this.store.dest.getName() + " cancelled: "
1315                            + (this.store.canceledTasks / this.store.doneTasks) * 100);
1316                    this.store.canceledTasks = this.store.doneTasks = 0;
1317                }
1318            } catch (Throwable t) {
1319                this.future.setException(t);
1320                removeQueueTask(this.store, this.message.getMessageId());
1321            }
1322        }
1323
1324        protected Message getMessage() {
1325            return this.message;
1326        }
1327
1328        private class InnerFutureTask extends FutureTask<Object> implements ListenableFuture<Object>  {
1329
1330            private Runnable listener;
1331            public InnerFutureTask(Runnable runnable) {
1332                super(runnable, null);
1333
1334            }
1335
1336            public void setException(final Throwable e) {
1337                super.setException(e);
1338            }
1339
1340            public void complete() {
1341                super.set(null);
1342            }
1343
1344            @Override
1345            public void done() {
1346                fireListener();
1347            }
1348
1349            @Override
1350            public void addListener(Runnable listener) {
1351                this.listener = listener;
1352                if (isDone()) {
1353                    fireListener();
1354                }
1355            }
1356
1357            private void fireListener() {
1358                if (listener != null) {
1359                    try {
1360                        listener.run();
1361                    } catch (Exception ignored) {
1362                        LOG.warn("Unexpected exception from future {} listener callback {}", this, listener, ignored);
1363                    }
1364                }
1365            }
1366        }
1367    }
1368
1369    class StoreTopicTask extends StoreQueueTask {
1370        private final int subscriptionCount;
1371        private final List<String> subscriptionKeys = new ArrayList<String>(1);
1372        private final KahaDBTopicMessageStore topicStore;
1373        public StoreTopicTask(KahaDBTopicMessageStore store, ConnectionContext context, Message message,
1374                int subscriptionCount) {
1375            super(store, context, message);
1376            this.topicStore = store;
1377            this.subscriptionCount = subscriptionCount;
1378
1379        }
1380
1381        @Override
1382        public void aquireLocks() {
1383            if (this.locked.compareAndSet(false, true)) {
1384                try {
1385                    globalTopicSemaphore.acquire();
1386                    store.acquireLocalAsyncLock();
1387                    message.incrementReferenceCount();
1388                } catch (InterruptedException e) {
1389                    LOG.warn("Failed to aquire lock", e);
1390                }
1391            }
1392        }
1393
1394        @Override
1395        public void releaseLocks() {
1396            if (this.locked.compareAndSet(true, false)) {
1397                message.decrementReferenceCount();
1398                store.releaseLocalAsyncLock();
1399                globalTopicSemaphore.release();
1400            }
1401        }
1402
1403        /**
1404         * add a key
1405         *
1406         * @param key
1407         * @return true if all acknowledgements received
1408         */
1409        public boolean addSubscriptionKey(String key) {
1410            synchronized (this.subscriptionKeys) {
1411                this.subscriptionKeys.add(key);
1412            }
1413            return this.subscriptionKeys.size() >= this.subscriptionCount;
1414        }
1415
1416        @Override
1417        public void run() {
1418            this.store.doneTasks++;
1419            try {
1420                if (this.done.compareAndSet(false, true)) {
1421                    this.topicStore.addMessage(context, message);
1422                    // apply any acks we have
1423                    synchronized (this.subscriptionKeys) {
1424                        for (String key : this.subscriptionKeys) {
1425                            this.topicStore.doAcknowledge(context, key, this.message.getMessageId(), null);
1426
1427                        }
1428                    }
1429                    removeTopicTask(this.topicStore, this.message.getMessageId());
1430                    this.future.complete();
1431                } else if (cancelledTaskModMetric > 0 && this.store.canceledTasks++ % cancelledTaskModMetric == 0) {
1432                    System.err.println(this.store.dest.getName() + " cancelled: "
1433                            + (this.store.canceledTasks / this.store.doneTasks) * 100);
1434                    this.store.canceledTasks = this.store.doneTasks = 0;
1435                }
1436            } catch (Throwable t) {
1437                this.future.setException(t);
1438                removeTopicTask(this.topicStore, this.message.getMessageId());
1439            }
1440        }
1441    }
1442
1443    public class StoreTaskExecutor extends ThreadPoolExecutor {
1444
1445        public StoreTaskExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit timeUnit, BlockingQueue<Runnable> queue, ThreadFactory threadFactory) {
1446            super(corePoolSize, maximumPoolSize, keepAliveTime, timeUnit, queue, threadFactory);
1447        }
1448
1449        @Override
1450        protected void afterExecute(Runnable runnable, Throwable throwable) {
1451            super.afterExecute(runnable, throwable);
1452
1453            if (runnable instanceof StoreTask) {
1454               ((StoreTask)runnable).releaseLocks();
1455            }
1456        }
1457    }
1458
1459    @Override
1460    public JobSchedulerStore createJobSchedulerStore() throws IOException, UnsupportedOperationException {
1461        return new JobSchedulerStoreImpl();
1462    }
1463}