Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.broker.region.cursors;
018
019import java.util.Iterator;
020import java.util.LinkedList;
021import java.util.ListIterator;
022import java.util.concurrent.CancellationException;
023import java.util.concurrent.Future;
024import java.util.concurrent.TimeUnit;
025import java.util.concurrent.TimeoutException;
026import org.apache.activemq.broker.region.Destination;
027import org.apache.activemq.broker.region.MessageReference;
028import org.apache.activemq.broker.region.Subscription;
029import org.apache.activemq.command.Message;
030import org.apache.activemq.command.MessageId;
031import org.apache.activemq.store.MessageRecoveryListener;
032import org.slf4j.Logger;
033import org.slf4j.LoggerFactory;
034
035/**
036 *  Store based cursor
037 *
038 */
039public abstract class AbstractStoreCursor extends AbstractPendingMessageCursor implements MessageRecoveryListener {
040    private static final Logger LOG = LoggerFactory.getLogger(AbstractStoreCursor.class);
041    protected final Destination regionDestination;
042    protected final PendingList batchList;
043    private Iterator<MessageReference> iterator = null;
044    protected boolean batchResetNeeded = false;
045    protected int size;
046    private LinkedList<MessageId> pendingCachedIds = new LinkedList<>();
047    private static int SYNC_ADD = 0;
048    private static int ASYNC_ADD = 1;
049    final MessageId[] lastCachedIds = new MessageId[2];
050    protected boolean hadSpace = false;
051
052    protected AbstractStoreCursor(Destination destination) {
053        super((destination != null ? destination.isPrioritizedMessages():false));
054        this.regionDestination=destination;
055        if (this.prioritizedMessages) {
056            this.batchList= new PrioritizedPendingList();
057        } else {
058            this.batchList = new OrderedPendingList();
059        }
060    }
061
062
063    public final synchronized void start() throws Exception{
064        if (!isStarted()) {
065            super.start();
066            resetBatch();
067            resetSize();
068            setCacheEnabled(size==0&&useCache);
069        }
070    }
071
072    protected void resetSize() {
073        this.size = getStoreSize();
074    }
075
076    @Override
077    public void rebase() {
078        resetSize();
079    }
080
081    public final synchronized void stop() throws Exception {
082        resetBatch();
083        super.stop();
084        gc();
085    }
086
087
088    public final boolean recoverMessage(Message message) throws Exception {
089        return recoverMessage(message,false);
090    }
091
092    public synchronized boolean recoverMessage(Message message, boolean cached) throws Exception {
093        boolean recovered = false;
094        if (recordUniqueId(message.getMessageId())) {
095            if (!cached) {
096                message.setRegionDestination(regionDestination);
097                if( message.getMemoryUsage()==null ) {
098                    message.setMemoryUsage(this.getSystemUsage().getMemoryUsage());
099                }
100            }
101            message.incrementReferenceCount();
102            batchList.addMessageLast(message);
103            clearIterator(true);
104            recovered = true;
105        } else if (!cached) {
106            // a duplicate from the store (!cached) - needs to be removed/acked - otherwise it will get re dispatched on restart
107            if (duplicateFromStoreExcepted(message)) {
108                if (LOG.isTraceEnabled()) {
109                    LOG.trace("{} store replayed pending message due to concurrentStoreAndDispatchQueues {} seq: {}", this, message.getMessageId(), message.getMessageId().getFutureOrSequenceLong());
110                }
111            } else {
112                LOG.warn("{} - cursor got duplicate from store {} seq: {}", this, message.getMessageId(), message.getMessageId().getFutureOrSequenceLong());
113                duplicate(message);
114            }
115        } else {
116            LOG.warn("{} - cursor got duplicate send {} seq: {}", this, message.getMessageId(), message.getMessageId().getFutureOrSequenceLong());
117            if (gotToTheStore(message)) {
118                duplicate(message);
119            }
120        }
121        return recovered;
122    }
123
124    protected boolean duplicateFromStoreExcepted(Message message) {
125        // expected for messages pending acks with kahadb.concurrentStoreAndDispatchQueues=true for
126        // which this existing unused flag has been repurposed
127        return message.isRecievedByDFBridge();
128    }
129
130    public static boolean gotToTheStore(Message message) throws Exception {
131        if (message.isRecievedByDFBridge()) {
132            // concurrent store and dispatch - wait to see if the message gets to the store to see
133            // if the index suppressed it (original still present), or whether it was stored and needs to be removed
134            Object possibleFuture = message.getMessageId().getFutureOrSequenceLong();
135            if (possibleFuture instanceof Future) {
136                ((Future) possibleFuture).get();
137            }
138            // need to access again after wait on future
139            Object sequence = message.getMessageId().getFutureOrSequenceLong();
140            return (sequence != null && sequence instanceof Long && Long.compare((Long) sequence, -1l) != 0);
141        }
142        return true;
143    }
144
145    // track for processing outside of store index lock so we can dlq
146    final LinkedList<Message> duplicatesFromStore = new LinkedList<Message>();
147    private void duplicate(Message message) {
148        duplicatesFromStore.add(message);
149    }
150
151    void dealWithDuplicates() {
152        for (Message message : duplicatesFromStore) {
153            regionDestination.duplicateFromStore(message, getSubscription());
154        }
155        duplicatesFromStore.clear();
156    }
157
158    public final synchronized void reset() {
159        if (batchList.isEmpty()) {
160            try {
161                fillBatch();
162            } catch (Exception e) {
163                LOG.error("{} - Failed to fill batch", this, e);
164                throw new RuntimeException(e);
165            }
166        }
167        clearIterator(true);
168        size();
169    }
170
171
172    public synchronized void release() {
173        clearIterator(false);
174    }
175
176    private synchronized void clearIterator(boolean ensureIterator) {
177        boolean haveIterator = this.iterator != null;
178        this.iterator=null;
179        if(haveIterator&&ensureIterator) {
180            ensureIterator();
181        }
182    }
183
184    private synchronized void ensureIterator() {
185        if(this.iterator==null) {
186            this.iterator=this.batchList.iterator();
187        }
188    }
189
190
191    public final void finished() {
192    }
193
194
195    public final synchronized boolean hasNext() {
196        if (batchList.isEmpty()) {
197            try {
198                fillBatch();
199            } catch (Exception e) {
200                LOG.error("{} - Failed to fill batch", this, e);
201                throw new RuntimeException(e);
202            }
203        }
204        ensureIterator();
205        return this.iterator.hasNext();
206    }
207
208
209    public final synchronized MessageReference next() {
210        MessageReference result = null;
211        if (!this.batchList.isEmpty()&&this.iterator.hasNext()) {
212            result = this.iterator.next();
213        }
214        last = result;
215        if (result != null) {
216            result.incrementReferenceCount();
217        }
218        return result;
219    }
220
221    public synchronized boolean addMessageLast(MessageReference node) throws Exception {
222        boolean disableCache = false;
223        if (hasSpace()) {
224            if (isCacheEnabled()) {
225                if (recoverMessage(node.getMessage(),true)) {
226                    trackLastCached(node);
227                } else {
228                    dealWithDuplicates();
229                    return false;
230                }
231            }
232        } else {
233            disableCache = true;
234        }
235
236        if (disableCache && isCacheEnabled()) {
237            if (LOG.isTraceEnabled()) {
238                LOG.trace("{} - disabling cache on add {} {}", this, node.getMessageId(), node.getMessageId().getFutureOrSequenceLong());
239            }
240            syncWithStore(node.getMessage());
241            setCacheEnabled(false);
242        }
243        size++;
244        return true;
245    }
246
247    @Override
248    public synchronized boolean isCacheEnabled() {
249        return super.isCacheEnabled() || enableCacheNow();
250    }
251
252    protected boolean enableCacheNow() {
253        boolean result = false;
254        if (canEnableCash()) {
255            setCacheEnabled(true);
256            result = true;
257            if (LOG.isTraceEnabled()) {
258                LOG.trace("{} enabling cache on empty store", this);
259            }
260        }
261        return result;
262    }
263
264    protected boolean canEnableCash() {
265        return useCache && size==0 && hasSpace() && isStarted();
266    }
267
268    private void syncWithStore(Message currentAdd) throws Exception {
269        pruneLastCached();
270        for (ListIterator<MessageId> it = pendingCachedIds.listIterator(pendingCachedIds.size()); it.hasPrevious(); ) {
271            MessageId lastPending = it.previous();
272            Object futureOrLong = lastPending.getFutureOrSequenceLong();
273            if (futureOrLong instanceof Future) {
274                Future future = (Future) futureOrLong;
275                if (future.isCancelled()) {
276                    continue;
277                }
278                try {
279                    future.get(5, TimeUnit.SECONDS);
280                    setLastCachedId(ASYNC_ADD, lastPending);
281                } catch (CancellationException ok) {
282                    continue;
283                } catch (TimeoutException potentialDeadlock) {
284                    LOG.debug("{} timed out waiting for async add", this, potentialDeadlock);
285                } catch (Exception worstCaseWeReplay) {
286                    LOG.debug("{} exception waiting for async add", this, worstCaseWeReplay);
287                }
288            } else {
289                setLastCachedId(ASYNC_ADD, lastPending);
290            }
291            break;
292        }
293
294        MessageId candidate = lastCachedIds[ASYNC_ADD];
295        if (candidate != null) {
296            // ensure we don't skip current possibly sync add b/c we waited on the future
297            if (!isAsync(currentAdd) && Long.compare(((Long) currentAdd.getMessageId().getFutureOrSequenceLong()), ((Long) lastCachedIds[ASYNC_ADD].getFutureOrSequenceLong())) < 0) {
298                if (LOG.isTraceEnabled()) {
299                    LOG.trace("no set batch from async:" + candidate.getFutureOrSequenceLong() + " >= than current: " + currentAdd.getMessageId().getFutureOrSequenceLong() + ", " + this);
300                }
301                candidate = null;
302            }
303        }
304        if (candidate == null) {
305            candidate = lastCachedIds[SYNC_ADD];
306        }
307        if (candidate != null) {
308            setBatch(candidate);
309        }
310        // cleanup
311        lastCachedIds[SYNC_ADD] = lastCachedIds[ASYNC_ADD] = null;
312        pendingCachedIds.clear();
313    }
314
315    private void trackLastCached(MessageReference node) {
316        if (isAsync(node.getMessage())) {
317            pruneLastCached();
318            pendingCachedIds.add(node.getMessageId());
319        } else {
320            setLastCachedId(SYNC_ADD, node.getMessageId());
321        }
322    }
323
324    private static final boolean isAsync(Message message) {
325        return message.isRecievedByDFBridge() || message.getMessageId().getFutureOrSequenceLong() instanceof Future;
326    }
327
328    private void pruneLastCached() {
329        for (Iterator<MessageId> it = pendingCachedIds.iterator(); it.hasNext(); ) {
330            MessageId candidate = it.next();
331            final Object futureOrLong = candidate.getFutureOrSequenceLong();
332            if (futureOrLong instanceof Future) {
333                Future future = (Future) futureOrLong;
334                if (future.isCancelled()) {
335                    it.remove();
336                } else {
337                    // we don't want to wait for work to complete
338                    break;
339                }
340            } else {
341                // complete
342                setLastCachedId(ASYNC_ADD, candidate);
343
344                // keep lock step with sync adds while order is preserved
345                if (lastCachedIds[SYNC_ADD] != null) {
346                    long next = 1 + (Long)lastCachedIds[SYNC_ADD].getFutureOrSequenceLong();
347                    if (Long.compare((Long)futureOrLong, next) == 0) {
348                        setLastCachedId(SYNC_ADD, candidate);
349                    }
350                }
351                it.remove();
352            }
353        }
354    }
355
356    private void setLastCachedId(final int index, MessageId candidate) {
357        MessageId lastCacheId = lastCachedIds[index];
358        if (lastCacheId == null) {
359            lastCachedIds[index] = candidate;
360        } else {
361            Object lastCacheFutureOrSequenceLong = lastCacheId.getFutureOrSequenceLong();
362            Object candidateOrSequenceLong = candidate.getFutureOrSequenceLong();
363            if (lastCacheFutureOrSequenceLong == null) { // possibly null for topics
364                lastCachedIds[index] = candidate;
365            } else if (candidateOrSequenceLong != null &&
366                    Long.compare(((Long) candidateOrSequenceLong), ((Long) lastCacheFutureOrSequenceLong)) > 0) {
367                lastCachedIds[index] = candidate;
368            } if (LOG.isTraceEnabled()) {
369                LOG.trace("no set last cached[" + index + "] current:" + lastCacheFutureOrSequenceLong + " <= than candidate: " + candidateOrSequenceLong+ ", " + this);
370            }
371        }
372    }
373
374    protected void setBatch(MessageId messageId) throws Exception {
375    }
376
377
378    public synchronized void addMessageFirst(MessageReference node) throws Exception {
379        setCacheEnabled(false);
380        size++;
381    }
382
383
384    public final synchronized void remove() {
385        size--;
386        if (iterator!=null) {
387            iterator.remove();
388        }
389        if (last != null) {
390            last.decrementReferenceCount();
391        }
392    }
393
394
395    public final synchronized void remove(MessageReference node) {
396        if (batchList.remove(node) != null) {
397            size--;
398            setCacheEnabled(false);
399        }
400    }
401
402
403    public final synchronized void clear() {
404        gc();
405    }
406
407
408    public synchronized void gc() {
409        for (MessageReference msg : batchList) {
410            rollback(msg.getMessageId());
411            msg.decrementReferenceCount();
412        }
413        batchList.clear();
414        clearIterator(false);
415        batchResetNeeded = true;
416        setCacheEnabled(false);
417    }
418
419    protected final synchronized void fillBatch() {
420        if (LOG.isTraceEnabled()) {
421            LOG.trace("{} fillBatch", this);
422        }
423        if (batchResetNeeded) {
424            resetSize();
425            setMaxBatchSize(Math.min(regionDestination.getMaxPageSize(), size));
426            resetBatch();
427            this.batchResetNeeded = false;
428        }
429        if (this.batchList.isEmpty() && this.size >0) {
430            try {
431                doFillBatch();
432            } catch (Exception e) {
433                LOG.error("{} - Failed to fill batch", this, e);
434                throw new RuntimeException(e);
435            }
436        }
437    }
438
439
440    public final synchronized boolean isEmpty() {
441        // negative means more messages added to store through queue.send since last reset
442        return size == 0;
443    }
444
445
446    public final synchronized boolean hasMessagesBufferedToDeliver() {
447        return !batchList.isEmpty();
448    }
449
450
451    public final synchronized int size() {
452        if (size < 0) {
453            this.size = getStoreSize();
454        }
455        return size;
456    }
457
458    @Override
459    public String toString() {
460        return super.toString() + ":" + regionDestination.getActiveMQDestination().getPhysicalName() + ",batchResetNeeded=" + batchResetNeeded
461                    + ",size=" + this.size + ",cacheEnabled=" + isCacheEnabled()
462                    + ",maxBatchSize:" + maxBatchSize + ",hasSpace:" + hasSpace() + ",pendingCachedIds.size:" + pendingCachedIds.size()
463                    + ",lastSyncCachedId:" + lastCachedIds[SYNC_ADD] + ",lastSyncCachedId-seq:" + (lastCachedIds[SYNC_ADD] != null ? lastCachedIds[SYNC_ADD].getFutureOrSequenceLong() : "null")
464                    + ",lastAsyncCachedId:" + lastCachedIds[ASYNC_ADD] + ",lastAsyncCachedId-seq:" + (lastCachedIds[ASYNC_ADD] != null ? lastCachedIds[ASYNC_ADD].getFutureOrSequenceLong() : "null");
465    }
466
467    protected abstract void doFillBatch() throws Exception;
468
469    protected abstract void resetBatch();
470
471    protected abstract int getStoreSize();
472
473    protected abstract boolean isStoreEmpty();
474
475    public Subscription getSubscription() {
476        return null;
477    }
478}