Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.broker.region.cursors;
018
019import java.util.Iterator;
020import java.util.LinkedList;
021import java.util.ListIterator;
022import java.util.concurrent.CancellationException;
023import java.util.concurrent.Future;
024import java.util.concurrent.TimeUnit;
025import java.util.concurrent.TimeoutException;
026import org.apache.activemq.broker.region.Destination;
027import org.apache.activemq.broker.region.MessageReference;
028import org.apache.activemq.broker.region.Subscription;
029import org.apache.activemq.command.Message;
030import org.apache.activemq.command.MessageId;
031import org.apache.activemq.store.MessageRecoveryListener;
032import org.slf4j.Logger;
033import org.slf4j.LoggerFactory;
034
035/**
036 *  Store based cursor
037 *
038 */
039public abstract class AbstractStoreCursor extends AbstractPendingMessageCursor implements MessageRecoveryListener {
040    private static final Logger LOG = LoggerFactory.getLogger(AbstractStoreCursor.class);
041    protected final Destination regionDestination;
042    protected final PendingList batchList;
043    private Iterator<MessageReference> iterator = null;
044    protected boolean batchResetNeeded = false;
045    protected int size;
046    private LinkedList<MessageId> pendingCachedIds = new LinkedList<>();
047    private static int SYNC_ADD = 0;
048    private static int ASYNC_ADD = 1;
049    final MessageId[] lastCachedIds = new MessageId[2];
050    protected boolean hadSpace = false;
051
052    protected AbstractStoreCursor(Destination destination) {
053        super((destination != null ? destination.isPrioritizedMessages():false));
054        this.regionDestination=destination;
055        if (this.prioritizedMessages) {
056            this.batchList= new PrioritizedPendingList();
057        } else {
058            this.batchList = new OrderedPendingList();
059        }
060    }
061
062
063    public final synchronized void start() throws Exception{
064        if (!isStarted()) {
065            super.start();
066            resetBatch();
067            resetSize();
068            setCacheEnabled(size==0&&useCache);
069        }
070    }
071
072    protected void resetSize() {
073        this.size = getStoreSize();
074    }
075
076    @Override
077    public void rebase() {
078        resetSize();
079        MessageId lastAdded = lastCachedIds[SYNC_ADD];
080        if (lastAdded != null) {
081            try {
082                setBatch(lastAdded);
083            } catch (Exception e) {
084                LOG.error("{} - Failed to set batch on rebase", this, e);
085                throw new RuntimeException(e);
086            }
087        }
088    }
089
090    public final synchronized void stop() throws Exception {
091        resetBatch();
092        super.stop();
093        gc();
094    }
095
096
097    public final boolean recoverMessage(Message message) throws Exception {
098        return recoverMessage(message,false);
099    }
100
101    public synchronized boolean recoverMessage(Message message, boolean cached) throws Exception {
102        boolean recovered = false;
103        if (recordUniqueId(message.getMessageId())) {
104            if (!cached) {
105                message.setRegionDestination(regionDestination);
106                if( message.getMemoryUsage()==null ) {
107                    message.setMemoryUsage(this.getSystemUsage().getMemoryUsage());
108                }
109            }
110            message.incrementReferenceCount();
111            batchList.addMessageLast(message);
112            clearIterator(true);
113            recovered = true;
114        } else if (!cached) {
115            // a duplicate from the store (!cached) - needs to be removed/acked - otherwise it will get re dispatched on restart
116            if (duplicateFromStoreExcepted(message)) {
117                if (LOG.isTraceEnabled()) {
118                    LOG.trace("{} store replayed pending message due to concurrentStoreAndDispatchQueues {} seq: {}", this, message.getMessageId(), message.getMessageId().getFutureOrSequenceLong());
119                }
120            } else {
121                LOG.warn("{} - cursor got duplicate from store {} seq: {}", this, message.getMessageId(), message.getMessageId().getFutureOrSequenceLong());
122                duplicate(message);
123            }
124        } else {
125            LOG.warn("{} - cursor got duplicate send {} seq: {}", this, message.getMessageId(), message.getMessageId().getFutureOrSequenceLong());
126            if (gotToTheStore(message)) {
127                duplicate(message);
128            }
129        }
130        return recovered;
131    }
132
133    protected boolean duplicateFromStoreExcepted(Message message) {
134        // expected for messages pending acks with kahadb.concurrentStoreAndDispatchQueues=true for
135        // which this existing unused flag has been repurposed
136        return message.isRecievedByDFBridge();
137    }
138
139    public static boolean gotToTheStore(Message message) throws Exception {
140        if (message.isRecievedByDFBridge()) {
141            // concurrent store and dispatch - wait to see if the message gets to the store to see
142            // if the index suppressed it (original still present), or whether it was stored and needs to be removed
143            Object possibleFuture = message.getMessageId().getFutureOrSequenceLong();
144            if (possibleFuture instanceof Future) {
145                ((Future) possibleFuture).get();
146            }
147            // need to access again after wait on future
148            Object sequence = message.getMessageId().getFutureOrSequenceLong();
149            return (sequence != null && sequence instanceof Long && Long.compare((Long) sequence, -1l) != 0);
150        }
151        return true;
152    }
153
154    // track for processing outside of store index lock so we can dlq
155    final LinkedList<Message> duplicatesFromStore = new LinkedList<Message>();
156    private void duplicate(Message message) {
157        duplicatesFromStore.add(message);
158    }
159
160    void dealWithDuplicates() {
161        for (Message message : duplicatesFromStore) {
162            regionDestination.duplicateFromStore(message, getSubscription());
163        }
164        duplicatesFromStore.clear();
165    }
166
167    public final synchronized void reset() {
168        if (batchList.isEmpty()) {
169            try {
170                fillBatch();
171            } catch (Exception e) {
172                LOG.error("{} - Failed to fill batch", this, e);
173                throw new RuntimeException(e);
174            }
175        }
176        clearIterator(true);
177        size();
178    }
179
180
181    public synchronized void release() {
182        clearIterator(false);
183    }
184
185    private synchronized void clearIterator(boolean ensureIterator) {
186        boolean haveIterator = this.iterator != null;
187        this.iterator=null;
188        if(haveIterator&&ensureIterator) {
189            ensureIterator();
190        }
191    }
192
193    private synchronized void ensureIterator() {
194        if(this.iterator==null) {
195            this.iterator=this.batchList.iterator();
196        }
197    }
198
199
200    public final void finished() {
201    }
202
203
204    public final synchronized boolean hasNext() {
205        if (batchList.isEmpty()) {
206            try {
207                fillBatch();
208            } catch (Exception e) {
209                LOG.error("{} - Failed to fill batch", this, e);
210                throw new RuntimeException(e);
211            }
212        }
213        ensureIterator();
214        return this.iterator.hasNext();
215    }
216
217
218    public final synchronized MessageReference next() {
219        MessageReference result = null;
220        if (!this.batchList.isEmpty()&&this.iterator.hasNext()) {
221            result = this.iterator.next();
222        }
223        last = result;
224        if (result != null) {
225            result.incrementReferenceCount();
226        }
227        return result;
228    }
229
230    public synchronized boolean addMessageLast(MessageReference node) throws Exception {
231        boolean disableCache = false;
232        if (hasSpace()) {
233            if (isCacheEnabled()) {
234                if (recoverMessage(node.getMessage(),true)) {
235                    trackLastCached(node);
236                } else {
237                    dealWithDuplicates();
238                    return false;
239                }
240            }
241        } else {
242            disableCache = true;
243        }
244
245        if (disableCache && isCacheEnabled()) {
246            if (LOG.isTraceEnabled()) {
247                LOG.trace("{} - disabling cache on add {} {}", this, node.getMessageId(), node.getMessageId().getFutureOrSequenceLong());
248            }
249            syncWithStore(node.getMessage());
250            setCacheEnabled(false);
251        }
252        size++;
253        return true;
254    }
255
256    @Override
257    public synchronized boolean isCacheEnabled() {
258        return super.isCacheEnabled() || enableCacheNow();
259    }
260
261    protected boolean enableCacheNow() {
262        boolean result = false;
263        if (canEnableCash()) {
264            setCacheEnabled(true);
265            result = true;
266            if (LOG.isTraceEnabled()) {
267                LOG.trace("{} enabling cache on empty store", this);
268            }
269        }
270        return result;
271    }
272
273    protected boolean canEnableCash() {
274        return useCache && size==0 && hasSpace() && isStarted();
275    }
276
277    private void syncWithStore(Message currentAdd) throws Exception {
278        pruneLastCached();
279        for (ListIterator<MessageId> it = pendingCachedIds.listIterator(pendingCachedIds.size()); it.hasPrevious(); ) {
280            MessageId lastPending = it.previous();
281            Object futureOrLong = lastPending.getFutureOrSequenceLong();
282            if (futureOrLong instanceof Future) {
283                Future future = (Future) futureOrLong;
284                if (future.isCancelled()) {
285                    continue;
286                }
287                try {
288                    future.get(5, TimeUnit.SECONDS);
289                    setLastCachedId(ASYNC_ADD, lastPending);
290                } catch (CancellationException ok) {
291                    continue;
292                } catch (TimeoutException potentialDeadlock) {
293                    LOG.debug("{} timed out waiting for async add", this, potentialDeadlock);
294                } catch (Exception worstCaseWeReplay) {
295                    LOG.debug("{} exception waiting for async add", this, worstCaseWeReplay);
296                }
297            } else {
298                setLastCachedId(ASYNC_ADD, lastPending);
299            }
300            break;
301        }
302
303        MessageId candidate = lastCachedIds[ASYNC_ADD];
304        if (candidate != null) {
305            // ensure we don't skip current possibly sync add b/c we waited on the future
306            if (!isAsync(currentAdd) && Long.compare(((Long) currentAdd.getMessageId().getFutureOrSequenceLong()), ((Long) lastCachedIds[ASYNC_ADD].getFutureOrSequenceLong())) < 0) {
307                if (LOG.isTraceEnabled()) {
308                    LOG.trace("no set batch from async:" + candidate.getFutureOrSequenceLong() + " >= than current: " + currentAdd.getMessageId().getFutureOrSequenceLong() + ", " + this);
309                }
310                candidate = null;
311            }
312        }
313        if (candidate == null) {
314            candidate = lastCachedIds[SYNC_ADD];
315        }
316        if (candidate != null) {
317            setBatch(candidate);
318        }
319        // cleanup
320        lastCachedIds[SYNC_ADD] = lastCachedIds[ASYNC_ADD] = null;
321        pendingCachedIds.clear();
322    }
323
324    private void trackLastCached(MessageReference node) {
325        if (isAsync(node.getMessage())) {
326            pruneLastCached();
327            pendingCachedIds.add(node.getMessageId());
328        } else {
329            setLastCachedId(SYNC_ADD, node.getMessageId());
330        }
331    }
332
333    private static final boolean isAsync(Message message) {
334        return message.isRecievedByDFBridge() || message.getMessageId().getFutureOrSequenceLong() instanceof Future;
335    }
336
337    private void pruneLastCached() {
338        for (Iterator<MessageId> it = pendingCachedIds.iterator(); it.hasNext(); ) {
339            MessageId candidate = it.next();
340            final Object futureOrLong = candidate.getFutureOrSequenceLong();
341            if (futureOrLong instanceof Future) {
342                Future future = (Future) futureOrLong;
343                if (future.isCancelled()) {
344                    it.remove();
345                } else {
346                    // we don't want to wait for work to complete
347                    break;
348                }
349            } else {
350                // complete
351                setLastCachedId(ASYNC_ADD, candidate);
352
353                // keep lock step with sync adds while order is preserved
354                if (lastCachedIds[SYNC_ADD] != null) {
355                    long next = 1 + (Long)lastCachedIds[SYNC_ADD].getFutureOrSequenceLong();
356                    if (Long.compare((Long)futureOrLong, next) == 0) {
357                        setLastCachedId(SYNC_ADD, candidate);
358                    }
359                }
360                it.remove();
361            }
362        }
363    }
364
365    private void setLastCachedId(final int index, MessageId candidate) {
366        MessageId lastCacheId = lastCachedIds[index];
367        if (lastCacheId == null) {
368            lastCachedIds[index] = candidate;
369        } else {
370            Object lastCacheFutureOrSequenceLong = lastCacheId.getFutureOrSequenceLong();
371            Object candidateOrSequenceLong = candidate.getFutureOrSequenceLong();
372            if (lastCacheFutureOrSequenceLong == null) { // possibly null for topics
373                lastCachedIds[index] = candidate;
374            } else if (candidateOrSequenceLong != null &&
375                    Long.compare(((Long) candidateOrSequenceLong), ((Long) lastCacheFutureOrSequenceLong)) > 0) {
376                lastCachedIds[index] = candidate;
377            } if (LOG.isTraceEnabled()) {
378                LOG.trace("no set last cached[" + index + "] current:" + lastCacheFutureOrSequenceLong + " <= than candidate: " + candidateOrSequenceLong+ ", " + this);
379            }
380        }
381    }
382
383    protected void setBatch(MessageId messageId) throws Exception {
384    }
385
386
387    public synchronized void addMessageFirst(MessageReference node) throws Exception {
388        setCacheEnabled(false);
389        size++;
390    }
391
392
393    public final synchronized void remove() {
394        size--;
395        if (iterator!=null) {
396            iterator.remove();
397        }
398        if (last != null) {
399            last.decrementReferenceCount();
400        }
401    }
402
403
404    public final synchronized void remove(MessageReference node) {
405        if (batchList.remove(node) != null) {
406            size--;
407            setCacheEnabled(false);
408        }
409    }
410
411
412    public final synchronized void clear() {
413        gc();
414    }
415
416
417    public synchronized void gc() {
418        for (MessageReference msg : batchList) {
419            rollback(msg.getMessageId());
420            msg.decrementReferenceCount();
421        }
422        batchList.clear();
423        clearIterator(false);
424        batchResetNeeded = true;
425        setCacheEnabled(false);
426    }
427
428    protected final synchronized void fillBatch() {
429        if (LOG.isTraceEnabled()) {
430            LOG.trace("{} fillBatch", this);
431        }
432        if (batchResetNeeded) {
433            resetSize();
434            setMaxBatchSize(Math.min(regionDestination.getMaxPageSize(), size));
435            resetBatch();
436            this.batchResetNeeded = false;
437        }
438        if (this.batchList.isEmpty() && this.size >0) {
439            try {
440                doFillBatch();
441            } catch (Exception e) {
442                LOG.error("{} - Failed to fill batch", this, e);
443                throw new RuntimeException(e);
444            }
445        }
446    }
447
448
449    public final synchronized boolean isEmpty() {
450        // negative means more messages added to store through queue.send since last reset
451        return size == 0;
452    }
453
454
455    public final synchronized boolean hasMessagesBufferedToDeliver() {
456        return !batchList.isEmpty();
457    }
458
459
460    public final synchronized int size() {
461        if (size < 0) {
462            this.size = getStoreSize();
463        }
464        return size;
465    }
466
467    @Override
468    public String toString() {
469        return super.toString() + ":" + regionDestination.getActiveMQDestination().getPhysicalName() + ",batchResetNeeded=" + batchResetNeeded
470                    + ",size=" + this.size + ",cacheEnabled=" + isCacheEnabled()
471                    + ",maxBatchSize:" + maxBatchSize + ",hasSpace:" + hasSpace() + ",pendingCachedIds.size:" + pendingCachedIds.size()
472                    + ",lastSyncCachedId:" + lastCachedIds[SYNC_ADD] + ",lastSyncCachedId-seq:" + (lastCachedIds[SYNC_ADD] != null ? lastCachedIds[SYNC_ADD].getFutureOrSequenceLong() : "null")
473                    + ",lastAsyncCachedId:" + lastCachedIds[ASYNC_ADD] + ",lastAsyncCachedId-seq:" + (lastCachedIds[ASYNC_ADD] != null ? lastCachedIds[ASYNC_ADD].getFutureOrSequenceLong() : "null");
474    }
475
476    protected abstract void doFillBatch() throws Exception;
477
478    protected abstract void resetBatch();
479
480    protected abstract int getStoreSize();
481
482    protected abstract boolean isStoreEmpty();
483
484    public Subscription getSubscription() {
485        return null;
486    }
487}