Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.broker.region.cursors;
018
019import java.util.Iterator;
020import java.util.LinkedList;
021import java.util.ListIterator;
022import java.util.concurrent.CancellationException;
023import java.util.concurrent.ExecutionException;
024import java.util.concurrent.Future;
025import java.util.concurrent.TimeUnit;
026import java.util.concurrent.TimeoutException;
027
028import org.apache.activemq.broker.region.Destination;
029import org.apache.activemq.broker.region.MessageReference;
030import org.apache.activemq.broker.region.Subscription;
031import org.apache.activemq.command.Message;
032import org.apache.activemq.command.MessageId;
033import org.apache.activemq.store.MessageRecoveryListener;
034import org.slf4j.Logger;
035import org.slf4j.LoggerFactory;
036
037/**
038 *  Store based cursor
039 *
040 */
041public abstract class AbstractStoreCursor extends AbstractPendingMessageCursor implements MessageRecoveryListener {
042    private static final Logger LOG = LoggerFactory.getLogger(AbstractStoreCursor.class);
043    protected final Destination regionDestination;
044    protected final PendingList batchList;
045    private Iterator<MessageReference> iterator = null;
046    protected boolean batchResetNeeded = false;
047    protected int size;
048    private final LinkedList<MessageId> pendingCachedIds = new LinkedList<>();
049    private static int SYNC_ADD = 0;
050    private static int ASYNC_ADD = 1;
051    final MessageId[] lastCachedIds = new MessageId[2];
052    protected boolean hadSpace = false;
053
054
055
056    protected AbstractStoreCursor(Destination destination) {
057        super((destination != null ? destination.isPrioritizedMessages():false));
058        this.regionDestination=destination;
059        if (this.prioritizedMessages) {
060            this.batchList= new PrioritizedPendingList();
061        } else {
062            this.batchList = new OrderedPendingList();
063        }
064    }
065
066
067    @Override
068    public final synchronized void start() throws Exception{
069        if (!isStarted()) {
070            super.start();
071            resetBatch();
072            resetSize();
073            setCacheEnabled(size==0&&useCache);
074        }
075    }
076
077    protected void resetSize() {
078        this.size = getStoreSize();
079    }
080
081    @Override
082    public void rebase() {
083        MessageId lastAdded = lastCachedIds[SYNC_ADD];
084        if (lastAdded != null) {
085            try {
086                setBatch(lastAdded);
087            } catch (Exception e) {
088                LOG.error("{} - Failed to set batch on rebase", this, e);
089                throw new RuntimeException(e);
090            }
091        }
092    }
093
094    @Override
095    public final synchronized void stop() throws Exception {
096        resetBatch();
097        super.stop();
098        gc();
099    }
100
101
102    @Override
103    public final boolean recoverMessage(Message message) throws Exception {
104        return recoverMessage(message,false);
105    }
106
107    public synchronized boolean recoverMessage(Message message, boolean cached) throws Exception {
108        boolean recovered = false;
109        message.setRegionDestination(regionDestination);
110        if (recordUniqueId(message.getMessageId())) {
111            if (!cached) {
112                if( message.getMemoryUsage()==null ) {
113                    message.setMemoryUsage(this.getSystemUsage().getMemoryUsage());
114                }
115            }
116            message.incrementReferenceCount();
117            batchList.addMessageLast(message);
118            clearIterator(true);
119            recovered = true;
120        } else if (!cached) {
121            // a duplicate from the store (!cached) - needs to be removed/acked - otherwise it will get re dispatched on restart
122            if (duplicateFromStoreExcepted(message)) {
123                if (LOG.isTraceEnabled()) {
124                    LOG.trace("{} store replayed pending message due to concurrentStoreAndDispatchQueues {} seq: {}", this, message.getMessageId(), message.getMessageId().getFutureOrSequenceLong());
125                }
126            } else {
127                LOG.warn("{} - cursor got duplicate from store {} seq: {}", this, message.getMessageId(), message.getMessageId().getFutureOrSequenceLong());
128                duplicate(message);
129            }
130        } else {
131            LOG.warn("{} - cursor got duplicate send {} seq: {}", this, message.getMessageId(), message.getMessageId().getFutureOrSequenceLong());
132            if (gotToTheStore(message)) {
133                duplicate(message);
134            }
135        }
136        return recovered;
137    }
138
139    protected boolean duplicateFromStoreExcepted(Message message) {
140        // expected for messages pending acks with kahadb.concurrentStoreAndDispatchQueues=true for
141        // which this existing unused flag has been repurposed
142        return message.isRecievedByDFBridge();
143    }
144
145    public static boolean gotToTheStore(Message message) throws Exception {
146        if (message.isRecievedByDFBridge()) {
147            // concurrent store and dispatch - wait to see if the message gets to the store to see
148            // if the index suppressed it (original still present), or whether it was stored and needs to be removed
149            Object possibleFuture = message.getMessageId().getFutureOrSequenceLong();
150            if (possibleFuture instanceof Future) {
151                try {
152                    ((Future) possibleFuture).get();
153                } catch (Exception okToErrorOrCancelStoreOp) {}
154            }
155            // need to access again after wait on future
156            Object sequence = message.getMessageId().getFutureOrSequenceLong();
157            return (sequence != null && sequence instanceof Long && Long.compare((Long) sequence, -1l) != 0);
158        }
159        return true;
160    }
161
162    // track for processing outside of store index lock so we can dlq
163    final LinkedList<Message> duplicatesFromStore = new LinkedList<Message>();
164    private void duplicate(Message message) {
165        duplicatesFromStore.add(message);
166    }
167
168    void dealWithDuplicates() {
169        for (Message message : duplicatesFromStore) {
170            regionDestination.duplicateFromStore(message, getSubscription());
171        }
172        duplicatesFromStore.clear();
173    }
174
175    @Override
176    public final synchronized void reset() {
177        if (batchList.isEmpty()) {
178            try {
179                fillBatch();
180            } catch (Exception e) {
181                LOG.error("{} - Failed to fill batch", this, e);
182                throw new RuntimeException(e);
183            }
184        }
185        clearIterator(true);
186        size();
187    }
188
189
190    @Override
191    public synchronized void release() {
192        clearIterator(false);
193    }
194
195    private synchronized void clearIterator(boolean ensureIterator) {
196        boolean haveIterator = this.iterator != null;
197        this.iterator=null;
198        if(haveIterator&&ensureIterator) {
199            ensureIterator();
200        }
201    }
202
203    private synchronized void ensureIterator() {
204        if(this.iterator==null) {
205            this.iterator=this.batchList.iterator();
206        }
207    }
208
209
210    public final void finished() {
211    }
212
213
214    @Override
215    public final synchronized boolean hasNext() {
216        if (batchList.isEmpty()) {
217            try {
218                fillBatch();
219            } catch (Exception e) {
220                LOG.error("{} - Failed to fill batch", this, e);
221                throw new RuntimeException(e);
222            }
223        }
224        ensureIterator();
225        return this.iterator.hasNext();
226    }
227
228
229    @Override
230    public final synchronized MessageReference next() {
231        MessageReference result = null;
232        if (!this.batchList.isEmpty()&&this.iterator.hasNext()) {
233            result = this.iterator.next();
234        }
235        last = result;
236        if (result != null) {
237            result.incrementReferenceCount();
238        }
239        return result;
240    }
241
242    @Override
243    public synchronized boolean tryAddMessageLast(MessageReference node, long wait) throws Exception {
244        boolean disableCache = false;
245        if (hasSpace()) {
246            if (isCacheEnabled()) {
247                if (recoverMessage(node.getMessage(),true)) {
248                    trackLastCached(node);
249                } else {
250                    dealWithDuplicates();
251                    return false;
252                }
253            }
254        } else {
255            disableCache = true;
256        }
257
258        if (disableCache && isCacheEnabled()) {
259            if (LOG.isTraceEnabled()) {
260                LOG.trace("{} - disabling cache on add {} {}", this, node.getMessageId(), node.getMessageId().getFutureOrSequenceLong());
261            }
262            syncWithStore(node.getMessage());
263            setCacheEnabled(false);
264        }
265        size++;
266        return true;
267    }
268
269    @Override
270    public synchronized boolean isCacheEnabled() {
271        return super.isCacheEnabled() || enableCacheNow();
272    }
273
274    protected boolean enableCacheNow() {
275        boolean result = false;
276        if (canEnableCash()) {
277            setCacheEnabled(true);
278            result = true;
279            if (LOG.isTraceEnabled()) {
280                LOG.trace("{} enabling cache on empty store", this);
281            }
282        }
283        return result;
284    }
285
286    protected boolean canEnableCash() {
287        return useCache && size==0 && hasSpace() && isStarted();
288    }
289
290    @Override
291    public boolean canRecoveryNextMessage() {
292        // Should be safe to recovery messages if the overall memory usage if < 90%
293        return parentHasSpace(90);
294    }
295
296    private void syncWithStore(Message currentAdd) throws Exception {
297        pruneLastCached();
298        for (ListIterator<MessageId> it = pendingCachedIds.listIterator(pendingCachedIds.size()); it.hasPrevious(); ) {
299            MessageId lastPending = it.previous();
300            Object futureOrLong = lastPending.getFutureOrSequenceLong();
301            if (futureOrLong instanceof Future) {
302                Future future = (Future) futureOrLong;
303                if (future.isCancelled()) {
304                    continue;
305                }
306                try {
307                    future.get(5, TimeUnit.SECONDS);
308                    setLastCachedId(ASYNC_ADD, lastPending);
309                } catch (CancellationException ok) {
310                    continue;
311                } catch (TimeoutException potentialDeadlock) {
312                    LOG.debug("{} timed out waiting for async add", this, potentialDeadlock);
313                } catch (Exception worstCaseWeReplay) {
314                    LOG.debug("{} exception waiting for async add", this, worstCaseWeReplay);
315                }
316            } else {
317                setLastCachedId(ASYNC_ADD, lastPending);
318            }
319            break;
320        }
321
322        MessageId candidate = lastCachedIds[ASYNC_ADD];
323        if (candidate != null) {
324            // ensure we don't skip current possibly sync add b/c we waited on the future
325            if (!isAsync(currentAdd) && Long.compare(((Long) currentAdd.getMessageId().getFutureOrSequenceLong()), ((Long) lastCachedIds[ASYNC_ADD].getFutureOrSequenceLong())) < 0) {
326                if (LOG.isTraceEnabled()) {
327                    LOG.trace("no set batch from async:" + candidate.getFutureOrSequenceLong() + " >= than current: " + currentAdd.getMessageId().getFutureOrSequenceLong() + ", " + this);
328                }
329                candidate = null;
330            }
331        }
332        if (candidate == null) {
333            candidate = lastCachedIds[SYNC_ADD];
334        }
335        if (candidate != null) {
336            setBatch(candidate);
337        }
338        // cleanup
339        lastCachedIds[SYNC_ADD] = lastCachedIds[ASYNC_ADD] = null;
340        pendingCachedIds.clear();
341    }
342
343    private void trackLastCached(MessageReference node) {
344        if (isAsync(node.getMessage())) {
345            pruneLastCached();
346            pendingCachedIds.add(node.getMessageId());
347        } else {
348            setLastCachedId(SYNC_ADD, node.getMessageId());
349        }
350    }
351
352    private static final boolean isAsync(Message message) {
353        return message.isRecievedByDFBridge() || message.getMessageId().getFutureOrSequenceLong() instanceof Future;
354    }
355
356    private void pruneLastCached() {
357        for (Iterator<MessageId> it = pendingCachedIds.iterator(); it.hasNext(); ) {
358            MessageId candidate = it.next();
359            final Object futureOrLong = candidate.getFutureOrSequenceLong();
360            if (futureOrLong instanceof Future) {
361                Future future = (Future) futureOrLong;
362                if (future.isDone()) {
363                    if (future.isCancelled()) {
364                        it.remove();
365                    } else {
366                        // check for exception, we may be seeing old state
367                        try {
368                            future.get(0, TimeUnit.SECONDS);
369                            // stale; if we get a result next prune will see Long
370                        } catch (ExecutionException expected) {
371                            it.remove();
372                        } catch (Exception unexpected) {
373                            LOG.debug("{} unexpected exception verifying exception state of future", this, unexpected);
374                        }
375                    }
376                } else {
377                    // we don't want to wait for work to complete
378                    break;
379                }
380            } else {
381                // complete
382                setLastCachedId(ASYNC_ADD, candidate);
383
384                // keep lock step with sync adds while order is preserved
385                if (lastCachedIds[SYNC_ADD] != null) {
386                    long next = 1 + (Long)lastCachedIds[SYNC_ADD].getFutureOrSequenceLong();
387                    if (Long.compare((Long)futureOrLong, next) == 0) {
388                        setLastCachedId(SYNC_ADD, candidate);
389                    }
390                }
391                it.remove();
392            }
393        }
394    }
395
396    private void setLastCachedId(final int index, MessageId candidate) {
397        MessageId lastCacheId = lastCachedIds[index];
398        if (lastCacheId == null) {
399            lastCachedIds[index] = candidate;
400        } else {
401            Object lastCacheFutureOrSequenceLong = lastCacheId.getFutureOrSequenceLong();
402            Object candidateOrSequenceLong = candidate.getFutureOrSequenceLong();
403            if (lastCacheFutureOrSequenceLong == null) { // possibly null for topics
404                lastCachedIds[index] = candidate;
405            } else if (candidateOrSequenceLong != null &&
406                    Long.compare(((Long) candidateOrSequenceLong), ((Long) lastCacheFutureOrSequenceLong)) > 0) {
407                lastCachedIds[index] = candidate;
408            } else if (LOG.isTraceEnabled()) {
409                LOG.trace("no set last cached[" + index + "] current:" + lastCacheFutureOrSequenceLong + " <= than candidate: " + candidateOrSequenceLong+ ", " + this);
410            }
411        }
412    }
413
414    protected void setBatch(MessageId messageId) throws Exception {
415    }
416
417
418    @Override
419    public synchronized void addMessageFirst(MessageReference node) throws Exception {
420        size++;
421    }
422
423
424    @Override
425    public final synchronized void remove() {
426        size--;
427        if (iterator!=null) {
428            iterator.remove();
429        }
430        if (last != null) {
431            last.decrementReferenceCount();
432        }
433    }
434
435
436    @Override
437    public final synchronized void remove(MessageReference node) {
438        if (batchList.remove(node) != null) {
439            size--;
440            setCacheEnabled(false);
441        }
442    }
443
444
445    @Override
446    public final synchronized void clear() {
447        gc();
448    }
449
450
451    @Override
452    public synchronized void gc() {
453        for (MessageReference msg : batchList) {
454            rollback(msg.getMessageId());
455            msg.decrementReferenceCount();
456        }
457        batchList.clear();
458        clearIterator(false);
459        batchResetNeeded = true;
460        setCacheEnabled(false);
461    }
462
463    @Override
464    protected final synchronized void fillBatch() {
465        if (LOG.isTraceEnabled()) {
466            LOG.trace("{} fillBatch", this);
467        }
468        if (batchResetNeeded) {
469            resetSize();
470            setMaxBatchSize(Math.min(regionDestination.getMaxPageSize(), size));
471            resetBatch();
472            this.batchResetNeeded = false;
473        }
474        if (this.batchList.isEmpty() && this.size >0) {
475            try {
476                doFillBatch();
477            } catch (Exception e) {
478                LOG.error("{} - Failed to fill batch", this, e);
479                throw new RuntimeException(e);
480            }
481        }
482    }
483
484
485    @Override
486    public final synchronized boolean isEmpty() {
487        // negative means more messages added to store through queue.send since last reset
488        return size == 0;
489    }
490
491
492    @Override
493    public final synchronized boolean hasMessagesBufferedToDeliver() {
494        return !batchList.isEmpty();
495    }
496
497
498    @Override
499    public final synchronized int size() {
500        if (size < 0) {
501            this.size = getStoreSize();
502        }
503        return size;
504    }
505
506    @Override
507    public final synchronized long messageSize() {
508        return getStoreMessageSize();
509    }
510
511    @Override
512    public String toString() {
513        return super.toString() + ":" + regionDestination.getActiveMQDestination().getPhysicalName() + ",batchResetNeeded=" + batchResetNeeded
514                    + ",size=" + this.size + ",cacheEnabled=" + cacheEnabled
515                    + ",maxBatchSize:" + maxBatchSize + ",hasSpace:" + hasSpace() + ",pendingCachedIds.size:" + pendingCachedIds.size()
516                    + ",lastSyncCachedId:" + lastCachedIds[SYNC_ADD] + ",lastSyncCachedId-seq:" + (lastCachedIds[SYNC_ADD] != null ? lastCachedIds[SYNC_ADD].getFutureOrSequenceLong() : "null")
517                    + ",lastAsyncCachedId:" + lastCachedIds[ASYNC_ADD] + ",lastAsyncCachedId-seq:" + (lastCachedIds[ASYNC_ADD] != null ? lastCachedIds[ASYNC_ADD].getFutureOrSequenceLong() : "null");
518    }
519
520    protected abstract void doFillBatch() throws Exception;
521
522    protected abstract void resetBatch();
523
524    protected abstract int getStoreSize();
525
526    protected abstract long getStoreMessageSize();
527
528    protected abstract boolean isStoreEmpty();
529
530    public Subscription getSubscription() {
531        return null;
532    }
533}