001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.network;
018
019import java.io.IOException;
020import java.security.GeneralSecurityException;
021import java.security.cert.X509Certificate;
022import java.util.Arrays;
023import java.util.Collection;
024import java.util.Collections;
025import java.util.Iterator;
026import java.util.List;
027import java.util.Properties;
028import java.util.Set;
029import java.util.concurrent.ConcurrentHashMap;
030import java.util.concurrent.ConcurrentMap;
031import java.util.concurrent.CountDownLatch;
032import java.util.concurrent.ExecutionException;
033import java.util.concurrent.ExecutorService;
034import java.util.concurrent.Executors;
035import java.util.concurrent.Future;
036import java.util.concurrent.TimeUnit;
037import java.util.concurrent.TimeoutException;
038import java.util.concurrent.atomic.AtomicBoolean;
039import java.util.regex.Pattern;
040
041import javax.management.ObjectName;
042
043import org.apache.activemq.DestinationDoesNotExistException;
044import org.apache.activemq.Service;
045import org.apache.activemq.advisory.AdvisoryBroker;
046import org.apache.activemq.advisory.AdvisorySupport;
047import org.apache.activemq.broker.BrokerService;
048import org.apache.activemq.broker.BrokerServiceAware;
049import org.apache.activemq.broker.ConnectionContext;
050import org.apache.activemq.broker.TransportConnection;
051import org.apache.activemq.broker.region.AbstractRegion;
052import org.apache.activemq.broker.region.DurableTopicSubscription;
053import org.apache.activemq.broker.region.Region;
054import org.apache.activemq.broker.region.RegionBroker;
055import org.apache.activemq.broker.region.Subscription;
056import org.apache.activemq.broker.region.policy.PolicyEntry;
057import org.apache.activemq.command.ActiveMQDestination;
058import org.apache.activemq.command.ActiveMQMessage;
059import org.apache.activemq.command.ActiveMQTempDestination;
060import org.apache.activemq.command.ActiveMQTopic;
061import org.apache.activemq.command.BrokerId;
062import org.apache.activemq.command.BrokerInfo;
063import org.apache.activemq.command.BrokerSubscriptionInfo;
064import org.apache.activemq.command.Command;
065import org.apache.activemq.command.CommandTypes;
066import org.apache.activemq.command.ConnectionError;
067import org.apache.activemq.command.ConnectionId;
068import org.apache.activemq.command.ConnectionInfo;
069import org.apache.activemq.command.ConsumerId;
070import org.apache.activemq.command.ConsumerInfo;
071import org.apache.activemq.command.DataStructure;
072import org.apache.activemq.command.DestinationInfo;
073import org.apache.activemq.command.ExceptionResponse;
074import org.apache.activemq.command.KeepAliveInfo;
075import org.apache.activemq.command.Message;
076import org.apache.activemq.command.MessageAck;
077import org.apache.activemq.command.MessageDispatch;
078import org.apache.activemq.command.MessageId;
079import org.apache.activemq.command.NetworkBridgeFilter;
080import org.apache.activemq.command.ProducerInfo;
081import org.apache.activemq.command.RemoveInfo;
082import org.apache.activemq.command.RemoveSubscriptionInfo;
083import org.apache.activemq.command.Response;
084import org.apache.activemq.command.SessionInfo;
085import org.apache.activemq.command.ShutdownInfo;
086import org.apache.activemq.command.SubscriptionInfo;
087import org.apache.activemq.command.WireFormatInfo;
088import org.apache.activemq.filter.DestinationFilter;
089import org.apache.activemq.filter.NonCachedMessageEvaluationContext;
090import org.apache.activemq.security.SecurityContext;
091import org.apache.activemq.transport.DefaultTransportListener;
092import org.apache.activemq.transport.FutureResponse;
093import org.apache.activemq.transport.ResponseCallback;
094import org.apache.activemq.transport.Transport;
095import org.apache.activemq.transport.TransportDisposedIOException;
096import org.apache.activemq.transport.TransportFilter;
097import org.apache.activemq.transport.failover.FailoverTransport;
098import org.apache.activemq.transport.tcp.TcpTransport;
099import org.apache.activemq.util.IdGenerator;
100import org.apache.activemq.util.IntrospectionSupport;
101import org.apache.activemq.util.LongSequenceGenerator;
102import org.apache.activemq.util.MarshallingSupport;
103import org.apache.activemq.util.NetworkBridgeUtils;
104import org.apache.activemq.util.ServiceStopper;
105import org.apache.activemq.util.ServiceSupport;
106import org.apache.activemq.util.StringToListOfActiveMQDestinationConverter;
107import org.slf4j.Logger;
108import org.slf4j.LoggerFactory;
109
110/**
111 * A useful base class for implementing demand forwarding bridges.
112 */
113public abstract class DemandForwardingBridgeSupport implements NetworkBridge, BrokerServiceAware {
114    private static final Logger LOG = LoggerFactory.getLogger(DemandForwardingBridgeSupport.class);
115    protected static final String DURABLE_SUB_PREFIX = "NC-DS_";
116    protected final Transport localBroker;
117    protected final Transport remoteBroker;
118    protected IdGenerator idGenerator = new IdGenerator();
119    protected final LongSequenceGenerator consumerIdGenerator = new LongSequenceGenerator();
120    protected ConnectionInfo localConnectionInfo;
121    protected ConnectionInfo remoteConnectionInfo;
122    protected SessionInfo localSessionInfo;
123    protected ProducerInfo producerInfo;
124    protected String remoteBrokerName = "Unknown";
125    protected String localClientId;
126    protected ConsumerInfo demandConsumerInfo;
127    protected int demandConsumerDispatched;
128    protected final AtomicBoolean localBridgeStarted = new AtomicBoolean(false);
129    protected final AtomicBoolean remoteBridgeStarted = new AtomicBoolean(false);
130    protected final AtomicBoolean bridgeFailed = new AtomicBoolean();
131    protected final AtomicBoolean disposed = new AtomicBoolean();
132    protected BrokerId localBrokerId;
133    protected ActiveMQDestination[] excludedDestinations;
134    protected ActiveMQDestination[] dynamicallyIncludedDestinations;
135    protected ActiveMQDestination[] staticallyIncludedDestinations;
136    protected ActiveMQDestination[] durableDestinations;
137    protected final ConcurrentMap<ConsumerId, DemandSubscription> subscriptionMapByLocalId = new ConcurrentHashMap<>();
138    protected final ConcurrentMap<ConsumerId, DemandSubscription> subscriptionMapByRemoteId = new ConcurrentHashMap<>();
139    protected final Set<ConsumerId> forcedDurableRemoteId = Collections.newSetFromMap(new ConcurrentHashMap<ConsumerId, Boolean>());
140    protected final BrokerId localBrokerPath[] = new BrokerId[]{null};
141    protected final CountDownLatch startedLatch = new CountDownLatch(2);
142    protected final CountDownLatch localStartedLatch = new CountDownLatch(1);
143    protected final CountDownLatch staticDestinationsLatch = new CountDownLatch(1);
144    protected final AtomicBoolean lastConnectSucceeded = new AtomicBoolean(false);
145    protected NetworkBridgeConfiguration configuration;
146    protected final NetworkBridgeFilterFactory defaultFilterFactory = new DefaultNetworkBridgeFilterFactory();
147
148    protected final BrokerId remoteBrokerPath[] = new BrokerId[]{null};
149    protected BrokerId remoteBrokerId;
150
151    protected final NetworkBridgeStatistics networkBridgeStatistics = new NetworkBridgeStatistics();
152
153    private NetworkBridgeListener networkBridgeListener;
154    private boolean createdByDuplex;
155    private BrokerInfo localBrokerInfo;
156    private BrokerInfo remoteBrokerInfo;
157
158    private final FutureBrokerInfo futureRemoteBrokerInfo = new FutureBrokerInfo(remoteBrokerInfo, disposed);
159    private final FutureBrokerInfo futureLocalBrokerInfo = new FutureBrokerInfo(localBrokerInfo, disposed);
160
161    private final AtomicBoolean started = new AtomicBoolean();
162    private TransportConnection duplexInitiatingConnection;
163    private final AtomicBoolean duplexInitiatingConnectionInfoReceived = new AtomicBoolean();
164    protected BrokerService brokerService = null;
165    private ObjectName mbeanObjectName;
166    private final ExecutorService serialExecutor = Executors.newSingleThreadExecutor();
167    //Use a new executor for processing BrokerSubscriptionInfo so we don't block other threads
168    private final ExecutorService syncExecutor = Executors.newSingleThreadExecutor();
169    private Transport duplexInboundLocalBroker = null;
170    private ProducerInfo duplexInboundLocalProducerInfo;
171
172    public DemandForwardingBridgeSupport(NetworkBridgeConfiguration configuration, Transport localBroker, Transport remoteBroker) {
173        this.configuration = configuration;
174        this.localBroker = localBroker;
175        this.remoteBroker = remoteBroker;
176    }
177
178    public void duplexStart(TransportConnection connection, BrokerInfo localBrokerInfo, BrokerInfo remoteBrokerInfo) throws Exception {
179        this.localBrokerInfo = localBrokerInfo;
180        this.remoteBrokerInfo = remoteBrokerInfo;
181        this.duplexInitiatingConnection = connection;
182        start();
183        serviceRemoteCommand(remoteBrokerInfo);
184    }
185
186    @Override
187    public void start() throws Exception {
188        if (started.compareAndSet(false, true)) {
189
190            if (brokerService == null) {
191                throw new IllegalArgumentException("BrokerService is null on " + this);
192            }
193
194            networkBridgeStatistics.setEnabled(brokerService.isEnableStatistics());
195
196            if (isDuplex()) {
197                duplexInboundLocalBroker = NetworkBridgeFactory.createLocalAsyncTransport(brokerService.getBroker().getVmConnectorURI());
198                duplexInboundLocalBroker.setTransportListener(new DefaultTransportListener() {
199
200                    @Override
201                    public void onCommand(Object o) {
202                        Command command = (Command) o;
203                        serviceLocalCommand(command);
204                    }
205
206                    @Override
207                    public void onException(IOException error) {
208                        serviceLocalException(error);
209                    }
210                });
211                duplexInboundLocalBroker.start();
212            }
213
214            localBroker.setTransportListener(new DefaultTransportListener() {
215
216                @Override
217                public void onCommand(Object o) {
218                    Command command = (Command) o;
219                    serviceLocalCommand(command);
220                }
221
222                @Override
223                public void onException(IOException error) {
224                    if (!futureLocalBrokerInfo.isDone()) {
225                        LOG.info("Error with pending local brokerInfo on: {} ({})", localBroker, error.getMessage());
226                        LOG.debug("Peer error: ", error);
227                        futureLocalBrokerInfo.cancel(true);
228                        return;
229                    }
230                    serviceLocalException(error);
231                }
232            });
233
234            remoteBroker.setTransportListener(new DefaultTransportListener() {
235
236                @Override
237                public void onCommand(Object o) {
238                    Command command = (Command) o;
239                    serviceRemoteCommand(command);
240                }
241
242                @Override
243                public void onException(IOException error) {
244                    if (!futureRemoteBrokerInfo.isDone()) {
245                        LOG.info("Error with pending remote brokerInfo on: {} ({})", remoteBroker, error.getMessage());
246                        LOG.debug("Peer error: ", error);
247                        futureRemoteBrokerInfo.cancel(true);
248                        return;
249                    }
250                    serviceRemoteException(error);
251                }
252            });
253
254            remoteBroker.start();
255            localBroker.start();
256
257            if (!disposed.get()) {
258                try {
259                    triggerStartAsyncNetworkBridgeCreation();
260                } catch (IOException e) {
261                    LOG.warn("Caught exception from remote start", e);
262                }
263            } else {
264                LOG.warn("Bridge was disposed before the start() method was fully executed.");
265                throw new TransportDisposedIOException();
266            }
267        }
268    }
269
270    @Override
271    public void stop() throws Exception {
272        if (started.compareAndSet(true, false)) {
273            if (disposed.compareAndSet(false, true)) {
274                LOG.debug(" stopping {} bridge to {}", configuration.getBrokerName(), remoteBrokerName);
275
276                futureRemoteBrokerInfo.cancel(true);
277                futureLocalBrokerInfo.cancel(true);
278
279                NetworkBridgeListener l = this.networkBridgeListener;
280                if (l != null) {
281                    l.onStop(this);
282                }
283                try {
284                    // local start complete
285                    if (startedLatch.getCount() < 2) {
286                        LOG.trace("{} unregister bridge ({}) to {}",
287                                configuration.getBrokerName(), this, remoteBrokerName);
288                        brokerService.getBroker().removeBroker(null, remoteBrokerInfo);
289                        brokerService.getBroker().networkBridgeStopped(remoteBrokerInfo);
290                    }
291
292                    remoteBridgeStarted.set(false);
293                    final CountDownLatch sendShutdown = new CountDownLatch(1);
294
295                    brokerService.getTaskRunnerFactory().execute(new Runnable() {
296                        @Override
297                        public void run() {
298                            try {
299                                serialExecutor.shutdown();
300                                if (!serialExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
301                                    List<Runnable> pendingTasks = serialExecutor.shutdownNow();
302                                    LOG.info("pending tasks on stop {}", pendingTasks);
303                                }
304                                //Shutdown the syncExecutor, call countDown to make sure a thread can
305                                //terminate if it is waiting
306                                staticDestinationsLatch.countDown();
307                                syncExecutor.shutdown();
308                                if (!syncExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
309                                    List<Runnable> pendingTasks = syncExecutor.shutdownNow();
310                                    LOG.info("pending tasks on stop {}", pendingTasks);
311                                }
312                                localBroker.oneway(new ShutdownInfo());
313                                remoteBroker.oneway(new ShutdownInfo());
314                            } catch (Throwable e) {
315                                LOG.debug("Caught exception sending shutdown", e);
316                            } finally {
317                                sendShutdown.countDown();
318                            }
319
320                        }
321                    }, "ActiveMQ ForwardingBridge StopTask");
322
323                    if (!sendShutdown.await(10, TimeUnit.SECONDS)) {
324                        LOG.info("Network Could not shutdown in a timely manner");
325                    }
326                } finally {
327                    ServiceStopper ss = new ServiceStopper();
328                    stopFailoverTransport(remoteBroker);
329                    ss.stop(remoteBroker);
330                    ss.stop(localBroker);
331                    ss.stop(duplexInboundLocalBroker);
332                    // Release the started Latch since another thread could be
333                    // stuck waiting for it to start up.
334                    startedLatch.countDown();
335                    startedLatch.countDown();
336                    localStartedLatch.countDown();
337                    staticDestinationsLatch.countDown();
338
339                    ss.throwFirstException();
340                }
341            }
342
343            LOG.info("{} bridge to {} stopped", configuration.getBrokerName(), remoteBrokerName);
344        }
345    }
346
347    private void stopFailoverTransport(Transport transport) {
348        FailoverTransport failoverTransport = transport.narrow(FailoverTransport.class);
349        if (failoverTransport != null) {
350            // may be blocked on write, in which case stop will block
351            try {
352                failoverTransport.handleTransportFailure(new IOException("Bridge stopped"));
353            } catch (InterruptedException ignored) {}
354        }
355    }
356
357    protected void triggerStartAsyncNetworkBridgeCreation() throws IOException {
358        brokerService.getTaskRunnerFactory().execute(new Runnable() {
359            @Override
360            public void run() {
361                final String originalName = Thread.currentThread().getName();
362                Thread.currentThread().setName("triggerStartAsyncNetworkBridgeCreation: " +
363                        "remoteBroker=" + remoteBroker + ", localBroker= " + localBroker);
364
365                try {
366                    // First we collect the info data from both the local and remote ends
367                    collectBrokerInfos();
368
369                    // Once we have all required broker info we can attempt to start
370                    // the local and then remote sides of the bridge.
371                    doStartLocalAndRemoteBridges();
372                } finally {
373                    Thread.currentThread().setName(originalName);
374                }
375            }
376        });
377    }
378
379    private void collectBrokerInfos() {
380        int timeout = 30000;
381        TcpTransport tcpTransport = remoteBroker.narrow(TcpTransport.class);
382        if (tcpTransport != null) {
383           timeout = tcpTransport.getConnectionTimeout();
384        }
385
386        // First wait for the remote to feed us its BrokerInfo, then we can check on
387        // the LocalBrokerInfo and decide is this is a loop.
388        try {
389            remoteBrokerInfo = futureRemoteBrokerInfo.get(timeout, TimeUnit.MILLISECONDS);
390            if (remoteBrokerInfo == null) {
391                serviceLocalException(new Throwable("remoteBrokerInfo is null"));
392                return;
393            }
394        } catch (Exception e) {
395            serviceRemoteException(e);
396            return;
397        }
398
399        try {
400            localBrokerInfo = futureLocalBrokerInfo.get(timeout, TimeUnit.MILLISECONDS);
401            if (localBrokerInfo == null) {
402                serviceLocalException(new Throwable("localBrokerInfo is null"));
403                return;
404            }
405
406            // Before we try and build the bridge lets check if we are in a loop
407            // and if so just stop now before registering anything.
408            remoteBrokerId = remoteBrokerInfo.getBrokerId();
409            if (localBrokerId.equals(remoteBrokerId)) {
410                LOG.trace("{} disconnecting remote loop back connector for: {}, with id: {}",
411                        configuration.getBrokerName(), remoteBrokerName, remoteBrokerId);
412                ServiceSupport.dispose(localBroker);
413                ServiceSupport.dispose(remoteBroker);
414                // the bridge is left in a bit of limbo, but it won't get retried
415                // in this state.
416                return;
417            }
418
419            // Fill in the remote broker's information now.
420            remoteBrokerPath[0] = remoteBrokerId;
421            remoteBrokerName = remoteBrokerInfo.getBrokerName();
422            if (configuration.isUseBrokerNamesAsIdSeed()) {
423                idGenerator = new IdGenerator(brokerService.getBrokerName() + "->" + remoteBrokerName);
424            }
425        } catch (Throwable e) {
426            serviceLocalException(e);
427        }
428    }
429
430    private void doStartLocalAndRemoteBridges() {
431
432        if (disposed.get()) {
433            return;
434        }
435
436        if (isCreatedByDuplex()) {
437            // apply remote (propagated) configuration to local duplex bridge before start
438            Properties props = null;
439            try {
440                props = MarshallingSupport.stringToProperties(remoteBrokerInfo.getNetworkProperties());
441                IntrospectionSupport.getProperties(configuration, props, null);
442                if (configuration.getExcludedDestinations() != null) {
443                    excludedDestinations = configuration.getExcludedDestinations().toArray(
444                            new ActiveMQDestination[configuration.getExcludedDestinations().size()]);
445                }
446                if (configuration.getStaticallyIncludedDestinations() != null) {
447                    staticallyIncludedDestinations = configuration.getStaticallyIncludedDestinations().toArray(
448                            new ActiveMQDestination[configuration.getStaticallyIncludedDestinations().size()]);
449                }
450                if (configuration.getDynamicallyIncludedDestinations() != null) {
451                    dynamicallyIncludedDestinations = configuration.getDynamicallyIncludedDestinations().toArray(
452                            new ActiveMQDestination[configuration.getDynamicallyIncludedDestinations().size()]);
453                }
454            } catch (Throwable t) {
455                LOG.error("Error mapping remote configuration: {}", props, t);
456            }
457        }
458
459        try {
460            startLocalBridge();
461        } catch (Throwable e) {
462            serviceLocalException(e);
463            return;
464        }
465
466        try {
467            startRemoteBridge();
468        } catch (Throwable e) {
469            serviceRemoteException(e);
470            return;
471        }
472
473        try {
474            if (safeWaitUntilStarted()) {
475                setupStaticDestinations();
476                staticDestinationsLatch.countDown();
477            }
478        } catch (Throwable e) {
479            serviceLocalException(e);
480        }
481    }
482
483    private void startLocalBridge() throws Throwable {
484        if (!bridgeFailed.get() && localBridgeStarted.compareAndSet(false, true)) {
485            synchronized (this) {
486                LOG.trace("{} starting local Bridge, localBroker={}", configuration.getBrokerName(), localBroker);
487                if (!disposed.get()) {
488
489                    if (idGenerator == null) {
490                        throw new IllegalStateException("Id Generator cannot be null");
491                    }
492
493                    localConnectionInfo = new ConnectionInfo();
494                    localConnectionInfo.setConnectionId(new ConnectionId(idGenerator.generateId()));
495                    localClientId = configuration.getName() + configuration.getClientIdToken() + remoteBrokerName + configuration.getClientIdToken() + "inbound" + configuration.getClientIdToken() + configuration.getBrokerName();
496                    localConnectionInfo.setClientId(localClientId);
497                    localConnectionInfo.setUserName(configuration.getUserName());
498                    localConnectionInfo.setPassword(configuration.getPassword());
499                    Transport originalTransport = remoteBroker;
500                    while (originalTransport instanceof TransportFilter) {
501                        originalTransport = ((TransportFilter) originalTransport).getNext();
502                    }
503                    if (originalTransport instanceof TcpTransport) {
504                        X509Certificate[] peerCerts = originalTransport.getPeerCertificates();
505                        localConnectionInfo.setTransportContext(peerCerts);
506                    }
507                    // sync requests that may fail
508                    Object resp = localBroker.request(localConnectionInfo);
509                    if (resp instanceof ExceptionResponse) {
510                        throw ((ExceptionResponse) resp).getException();
511                    }
512                    localSessionInfo = new SessionInfo(localConnectionInfo, 1);
513                    localBroker.oneway(localSessionInfo);
514
515                    if (configuration.isDuplex()) {
516                        // separate in-bound channel for forwards so we don't
517                        // contend with out-bound dispatch on same connection
518                        remoteBrokerInfo.setNetworkConnection(true);
519                        duplexInboundLocalBroker.oneway(remoteBrokerInfo);
520
521                        ConnectionInfo duplexLocalConnectionInfo = new ConnectionInfo();
522                        duplexLocalConnectionInfo.setConnectionId(new ConnectionId(idGenerator.generateId()));
523                        duplexLocalConnectionInfo.setClientId(configuration.getName() + configuration.getClientIdToken() + remoteBrokerName + configuration.getClientIdToken() + "inbound" + configuration.getClientIdToken() + "duplex"
524                                + configuration.getClientIdToken() + configuration.getBrokerName());
525                        duplexLocalConnectionInfo.setUserName(configuration.getUserName());
526                        duplexLocalConnectionInfo.setPassword(configuration.getPassword());
527
528                        if (originalTransport instanceof TcpTransport) {
529                            X509Certificate[] peerCerts = originalTransport.getPeerCertificates();
530                            duplexLocalConnectionInfo.setTransportContext(peerCerts);
531                        }
532                        // sync requests that may fail
533                        resp = duplexInboundLocalBroker.request(duplexLocalConnectionInfo);
534                        if (resp instanceof ExceptionResponse) {
535                            throw ((ExceptionResponse) resp).getException();
536                        }
537                        SessionInfo duplexInboundSession = new SessionInfo(duplexLocalConnectionInfo, 1);
538                        duplexInboundLocalProducerInfo = new ProducerInfo(duplexInboundSession, 1);
539                        duplexInboundLocalBroker.oneway(duplexInboundSession);
540                        duplexInboundLocalBroker.oneway(duplexInboundLocalProducerInfo);
541                    }
542                    brokerService.getBroker().networkBridgeStarted(remoteBrokerInfo, this.createdByDuplex, remoteBroker.toString());
543                    NetworkBridgeListener l = this.networkBridgeListener;
544                    if (l != null) {
545                        l.onStart(this);
546                    }
547
548                    // Let the local broker know the remote broker's ID.
549                    localBroker.oneway(remoteBrokerInfo);
550                    // new peer broker (a consumer can work with remote broker also)
551                    brokerService.getBroker().addBroker(null, remoteBrokerInfo);
552
553                    LOG.info("Network connection between {} and {} ({}) has been established.",
554                            localBroker, remoteBroker, remoteBrokerName);
555                    LOG.trace("{} register bridge ({}) to {}",
556                            configuration.getBrokerName(), this, remoteBrokerName);
557                } else {
558                    LOG.warn("Bridge was disposed before the startLocalBridge() method was fully executed.");
559                }
560                startedLatch.countDown();
561                localStartedLatch.countDown();
562            }
563        }
564    }
565
566    protected void startRemoteBridge() throws Exception {
567        if (!bridgeFailed.get() && remoteBridgeStarted.compareAndSet(false, true)) {
568            LOG.trace("{} starting remote Bridge, remoteBroker={}", configuration.getBrokerName(), remoteBroker);
569            synchronized (this) {
570                if (!isCreatedByDuplex()) {
571                    BrokerInfo brokerInfo = new BrokerInfo();
572                    brokerInfo.setBrokerName(configuration.getBrokerName());
573                    brokerInfo.setBrokerURL(configuration.getBrokerURL());
574                    brokerInfo.setNetworkConnection(true);
575                    brokerInfo.setDuplexConnection(configuration.isDuplex());
576                    // set our properties
577                    Properties props = new Properties();
578                    IntrospectionSupport.getProperties(configuration, props, null);
579
580                    String dynamicallyIncludedDestinationsKey = "dynamicallyIncludedDestinations";
581                    String staticallyIncludedDestinationsKey = "staticallyIncludedDestinations";
582
583                    if (!configuration.getDynamicallyIncludedDestinations().isEmpty()) {
584                        props.put(dynamicallyIncludedDestinationsKey,
585                                StringToListOfActiveMQDestinationConverter.
586                                convertFromActiveMQDestination(configuration.getDynamicallyIncludedDestinations(), true));
587                    }
588                    if (!configuration.getStaticallyIncludedDestinations().isEmpty()) {
589                        props.put(staticallyIncludedDestinationsKey,
590                                StringToListOfActiveMQDestinationConverter.
591                                convertFromActiveMQDestination(configuration.getStaticallyIncludedDestinations(), true));
592                    }
593
594                    props.remove("networkTTL");
595                    String str = MarshallingSupport.propertiesToString(props);
596                    brokerInfo.setNetworkProperties(str);
597                    brokerInfo.setBrokerId(this.localBrokerId);
598                    remoteBroker.oneway(brokerInfo);
599                    if (configuration.isSyncDurableSubs() &&
600                            remoteBroker.getWireFormat().getVersion() >= CommandTypes.PROTOCOL_VERSION_DURABLE_SYNC) {
601                        remoteBroker.oneway(NetworkBridgeUtils.getBrokerSubscriptionInfo(brokerService,
602                                configuration));
603                    }
604                }
605                if (remoteConnectionInfo != null) {
606                    remoteBroker.oneway(remoteConnectionInfo.createRemoveCommand());
607                }
608                remoteConnectionInfo = new ConnectionInfo();
609                remoteConnectionInfo.setConnectionId(new ConnectionId(idGenerator.generateId()));
610                remoteConnectionInfo.setClientId(configuration.getName() + configuration.getClientIdToken() + configuration.getBrokerName() + configuration.getClientIdToken() + "outbound");
611                
612                if(configuration.getRemoteUserName() != null) {
613                    remoteConnectionInfo.setUserName(configuration.getRemoteUserName());
614                    remoteConnectionInfo.setPassword(configuration.getRemotePassword());
615                } else {
616                    remoteConnectionInfo.setUserName(configuration.getUserName());
617                    remoteConnectionInfo.setPassword(configuration.getPassword());
618                }
619                remoteBroker.oneway(remoteConnectionInfo);
620
621                SessionInfo remoteSessionInfo = new SessionInfo(remoteConnectionInfo, 1);
622                remoteBroker.oneway(remoteSessionInfo);
623                producerInfo = new ProducerInfo(remoteSessionInfo, 1);
624                producerInfo.setResponseRequired(false);
625                remoteBroker.oneway(producerInfo);
626                // Listen to consumer advisory messages on the remote broker to determine demand.
627                if (!configuration.isStaticBridge()) {
628                    demandConsumerInfo = new ConsumerInfo(remoteSessionInfo, 1);
629                    // always dispatch advisory message asynchronously so that
630                    // we never block the producer broker if we are slow
631                    demandConsumerInfo.setDispatchAsync(true);
632                    String advisoryTopic = configuration.getDestinationFilter();
633                    if (configuration.isBridgeTempDestinations()) {
634                        advisoryTopic += "," + AdvisorySupport.TEMP_DESTINATION_COMPOSITE_ADVISORY_TOPIC;
635                    }
636                    demandConsumerInfo.setDestination(new ActiveMQTopic(advisoryTopic));
637                    configureConsumerPrefetch(demandConsumerInfo);
638                    remoteBroker.oneway(demandConsumerInfo);
639                }
640                startedLatch.countDown();
641            }
642        }
643    }
644
645    @Override
646    public void serviceRemoteException(Throwable error) {
647        if (!disposed.get()) {
648            if (error instanceof SecurityException || error instanceof GeneralSecurityException) {
649                LOG.error("Network connection between {} and {} shutdown due to a remote error: {}", localBroker, remoteBroker, error.toString());
650            } else {
651                LOG.warn("Network connection between {} and {} shutdown due to a remote error: {}", localBroker, remoteBroker, error.toString());
652            }
653            LOG.debug("The remote Exception was: {}", error, error);
654            brokerService.getTaskRunnerFactory().execute(new Runnable() {
655                @Override
656                public void run() {
657                    ServiceSupport.dispose(getControllingService());
658                }
659            });
660            fireBridgeFailed(error);
661        }
662    }
663
664    /**
665     * Checks whether or not this consumer is a direct bridge network subscription
666     * @param info
667     * @return
668     */
669    protected boolean isDirectBridgeConsumer(ConsumerInfo info) {
670        return (info.getSubscriptionName() != null && info.getSubscriptionName().startsWith(DURABLE_SUB_PREFIX)) &&
671                (info.getClientId() == null || info.getClientId().startsWith(configuration.getName()));
672    }
673
674    protected boolean isProxyBridgeSubscription(String clientId, String subName) {
675        if (subName != null && clientId != null) {
676            if (subName.startsWith(DURABLE_SUB_PREFIX) && !clientId.startsWith(configuration.getName())) {
677                return true;
678            }
679        }
680        return false;
681    }
682
683    /**
684     * This scenaior is primarily used for durable sync on broker restarts
685     *
686     * @param sub
687     * @param clientId
688     * @param subName
689     */
690    protected void addProxyNetworkSubscriptionClientId(final DemandSubscription sub, final String clientId, String subName) {
691        if (clientId != null && sub != null && subName != null) {
692                String newClientId = getProxyBridgeClientId(clientId);
693                final SubscriptionInfo newSubInfo = new SubscriptionInfo(newClientId, subName);
694                sub.getDurableRemoteSubs().add(newSubInfo);
695                LOG.debug("Adding proxy network subscription {} to demand subscription", newSubInfo);
696
697        } else {
698            LOG.debug("Skipping addProxyNetworkSubscription");
699        }
700    }
701
702    /**
703     * Add a durable remote proxy subscription when we can generate via the BrokerId path
704     * This is the most common scenario
705     *
706     * @param sub
707     * @param path
708     * @param subName
709     */
710    protected void addProxyNetworkSubscriptionBrokerPath(final DemandSubscription sub, final BrokerId[] path, String subName) {
711        if (sub != null && path.length > 1 && subName != null) {
712            String b1 = path[path.length-1].toString();
713            String b2 = path[path.length-2].toString();
714            final SubscriptionInfo newSubInfo = new SubscriptionInfo(b2 + configuration.getClientIdToken() + "inbound" + configuration.getClientIdToken() + b1, subName);
715            sub.getDurableRemoteSubs().add(newSubInfo);
716        }
717    }
718
719    private String getProxyBridgeClientId(String clientId) {
720        String newClientId = clientId;
721        String[] clientIdTokens = newClientId != null ? newClientId.split(Pattern.quote(configuration.getClientIdToken())) : null;
722        if (clientIdTokens != null && clientIdTokens.length > 2) {
723            newClientId = clientIdTokens[clientIdTokens.length - 3] +  configuration.getClientIdToken() + "inbound"
724                    + configuration.getClientIdToken() +  clientIdTokens[clientIdTokens.length -1];
725        }
726        return newClientId;
727    }
728
729    protected boolean isProxyNSConsumerBrokerPath(ConsumerInfo info) {
730        return info.getBrokerPath() != null && info.getBrokerPath().length > 1;
731    }
732
733    protected boolean isProxyNSConsumerClientId(String clientId) {
734        return clientId != null && clientId.split(Pattern.quote(configuration.getClientIdToken())).length > 3;
735    }
736
737    protected void serviceRemoteCommand(Command command) {
738        if (!disposed.get()) {
739            try {
740                if (command.isMessageDispatch()) {
741                    safeWaitUntilStarted();
742                    MessageDispatch md = (MessageDispatch) command;
743                    serviceRemoteConsumerAdvisory(md.getMessage().getDataStructure());
744                    ackAdvisory(md.getMessage());
745                } else if (command.isBrokerInfo()) {
746                    futureRemoteBrokerInfo.set((BrokerInfo) command);
747                } else if (command instanceof BrokerSubscriptionInfo) {
748                    final BrokerSubscriptionInfo brokerSubscriptionInfo = (BrokerSubscriptionInfo) command;
749
750                    //Start in a new thread so we don't block the transport waiting for staticDestinations
751                    syncExecutor.execute(new Runnable() {
752
753                        @Override
754                        public void run() {
755                            try {
756                                staticDestinationsLatch.await();
757                                //Make sure after the countDown of staticDestinationsLatch we aren't stopping
758                                if (!disposed.get()) {
759                                    BrokerSubscriptionInfo subInfo = brokerSubscriptionInfo;
760                                    LOG.debug("Received Remote BrokerSubscriptionInfo on {} from {}",
761                                            brokerService.getBrokerName(), subInfo.getBrokerName());
762
763                                    if (configuration.isSyncDurableSubs() && configuration.isConduitSubscriptions()
764                                            && !configuration.isDynamicOnly()) {
765                                        if (started.get()) {
766                                            if (subInfo.getSubscriptionInfos() != null) {
767                                                for (ConsumerInfo info : subInfo.getSubscriptionInfos()) {
768                                                    //re-add any process any non-NC consumers that match the
769                                                    //dynamicallyIncludedDestinations list
770                                                    //Also re-add network consumers that are not part of this direct
771                                                    //bridge (proxy of proxy bridges)
772                                                    if((info.getSubscriptionName() == null || !isDirectBridgeConsumer(info)) &&
773                                                            NetworkBridgeUtils.matchesDestinations(dynamicallyIncludedDestinations, info.getDestination())) {
774                                                        serviceRemoteConsumerAdvisory(info);
775                                                    }
776                                                }
777                                            }
778
779                                            //After re-added, clean up any empty durables
780                                            for (Iterator<DemandSubscription> i = subscriptionMapByLocalId.values().iterator(); i.hasNext(); ) {
781                                                DemandSubscription ds = i.next();
782                                                if (NetworkBridgeUtils.matchesDestinations(dynamicallyIncludedDestinations, ds.getLocalInfo().getDestination())) {
783                                                    cleanupDurableSub(ds, i);
784                                                }
785                                            }
786                                        }
787                                    }
788                                }
789                            } catch (Exception e) {
790                                LOG.warn("Error processing BrokerSubscriptionInfo: {}", e.getMessage(), e);
791                                LOG.debug(e.getMessage(), e);
792                            }
793                        }
794                    });
795
796                } else if (command.getClass() == ConnectionError.class) {
797                    ConnectionError ce = (ConnectionError) command;
798                    serviceRemoteException(ce.getException());
799                } else {
800                    if (isDuplex()) {
801                        LOG.trace("{} duplex command type: {}", configuration.getBrokerName(), command.getDataStructureType());
802                        if (command.isMessage()) {
803                            final ActiveMQMessage message = (ActiveMQMessage) command;
804                            if (NetworkBridgeFilter.isAdvisoryInterpretedByNetworkBridge(message)) {
805                                serviceRemoteConsumerAdvisory(message.getDataStructure());
806                                ackAdvisory(message);
807                            } else {
808                                if (!isPermissableDestination(message.getDestination(), true)) {
809                                    return;
810                                }
811                                safeWaitUntilStarted();
812                                // message being forwarded - we need to
813                                // propagate the response to our local send
814                                if (canDuplexDispatch(message)) {
815                                    message.setProducerId(duplexInboundLocalProducerInfo.getProducerId());
816                                    if (message.isResponseRequired() || configuration.isAlwaysSyncSend()) {
817                                        duplexInboundLocalBroker.asyncRequest(message, new ResponseCallback() {
818                                            final int correlationId = message.getCommandId();
819
820                                            @Override
821                                            public void onCompletion(FutureResponse resp) {
822                                                try {
823                                                    Response reply = resp.getResult();
824                                                    reply.setCorrelationId(correlationId);
825                                                    remoteBroker.oneway(reply);
826                                                    //increment counter when messages are received in duplex mode
827                                                    networkBridgeStatistics.getReceivedCount().increment();
828                                                } catch (IOException error) {
829                                                    LOG.error("Exception: {} on duplex forward of: {}", error, message);
830                                                    serviceRemoteException(error);
831                                                }
832                                            }
833                                        });
834                                    } else {
835                                        duplexInboundLocalBroker.oneway(message);
836                                        networkBridgeStatistics.getReceivedCount().increment();
837                                    }
838                                    serviceInboundMessage(message);
839                                } else {
840                                    if (message.isResponseRequired() || configuration.isAlwaysSyncSend()) {
841                                        Response reply = new Response();
842                                        reply.setCorrelationId(message.getCommandId());
843                                        remoteBroker.oneway(reply);
844                                    }
845                                }
846                            }
847                        } else {
848                            switch (command.getDataStructureType()) {
849                                case ConnectionInfo.DATA_STRUCTURE_TYPE:
850                                    if (duplexInitiatingConnection != null && duplexInitiatingConnectionInfoReceived.compareAndSet(false, true)) {
851                                        // end of initiating connection setup - propogate to initial connection to get mbean by clientid
852                                        duplexInitiatingConnection.processAddConnection((ConnectionInfo) command);
853                                    } else {
854                                        localBroker.oneway(command);
855                                    }
856                                    break;
857                                case SessionInfo.DATA_STRUCTURE_TYPE:
858                                    localBroker.oneway(command);
859                                    break;
860                                case ProducerInfo.DATA_STRUCTURE_TYPE:
861                                    // using duplexInboundLocalProducerInfo
862                                    break;
863                                case MessageAck.DATA_STRUCTURE_TYPE:
864                                    MessageAck ack = (MessageAck) command;
865                                    DemandSubscription localSub = subscriptionMapByRemoteId.get(ack.getConsumerId());
866                                    if (localSub != null) {
867                                        ack.setConsumerId(localSub.getLocalInfo().getConsumerId());
868                                        localBroker.oneway(ack);
869                                    } else {
870                                        LOG.warn("Matching local subscription not found for ack: {}", ack);
871                                    }
872                                    break;
873                                case ConsumerInfo.DATA_STRUCTURE_TYPE:
874                                    localStartedLatch.await();
875                                    if (started.get()) {
876                                        final ConsumerInfo consumerInfo = (ConsumerInfo) command;
877                                        if (isDuplicateSuppressionOff(consumerInfo)) {
878                                            addConsumerInfo(consumerInfo);
879                                        } else {
880                                            synchronized (brokerService.getVmConnectorURI()) {
881                                                addConsumerInfo(consumerInfo);
882                                            }
883                                        }
884                                    } else {
885                                        // received a subscription whilst stopping
886                                        LOG.warn("Stopping - ignoring ConsumerInfo: {}", command);
887                                    }
888                                    break;
889                                case ShutdownInfo.DATA_STRUCTURE_TYPE:
890                                    // initiator is shutting down, controlled case
891                                    // abortive close dealt with by inactivity monitor
892                                    LOG.info("Stopping network bridge on shutdown of remote broker");
893                                    serviceRemoteException(new IOException(command.toString()));
894                                    break;
895                                default:
896                                    LOG.debug("Ignoring remote command: {}", command);
897                            }
898                        }
899                    } else {
900                        switch (command.getDataStructureType()) {
901                            case KeepAliveInfo.DATA_STRUCTURE_TYPE:
902                            case WireFormatInfo.DATA_STRUCTURE_TYPE:
903                            case ShutdownInfo.DATA_STRUCTURE_TYPE:
904                                break;
905                            default:
906                                LOG.warn("Unexpected remote command: {}", command);
907                        }
908                    }
909                }
910            } catch (Throwable e) {
911                LOG.debug("Exception processing remote command: {}", command, e);
912                serviceRemoteException(e);
913            }
914        }
915    }
916
917    private void ackAdvisory(Message message) throws IOException {
918        demandConsumerDispatched++;
919        if (demandConsumerDispatched > (demandConsumerInfo.getPrefetchSize() *
920                (configuration.getAdvisoryAckPercentage() / 100f))) {
921            final MessageAck ack = new MessageAck(message, MessageAck.STANDARD_ACK_TYPE, demandConsumerDispatched);
922            ack.setConsumerId(demandConsumerInfo.getConsumerId());
923            brokerService.getTaskRunnerFactory().execute(new Runnable() {
924                @Override
925                public void run() {
926                    try {
927                        remoteBroker.oneway(ack);
928                    } catch (IOException e) {
929                        LOG.warn("Failed to send advisory ack {}", ack, e);
930                    }
931                }
932            });
933            demandConsumerDispatched = 0;
934        }
935    }
936
937    private void serviceRemoteConsumerAdvisory(DataStructure data) throws IOException {
938        final int networkTTL = configuration.getConsumerTTL();
939        if (data.getClass() == ConsumerInfo.class) {
940            // Create a new local subscription
941            ConsumerInfo info = (ConsumerInfo) data;
942            BrokerId[] path = info.getBrokerPath();
943
944            if (info.isBrowser()) {
945                LOG.debug("{} Ignoring sub from {}, browsers explicitly suppressed", configuration.getBrokerName(), remoteBrokerName);
946                return;
947            }
948
949            if (path != null && networkTTL > -1 && path.length >= networkTTL) {
950                LOG.debug("{} Ignoring sub from {}, restricted to {} network hops only: {}",
951                        configuration.getBrokerName(), remoteBrokerName, networkTTL, info);
952                return;
953            }
954
955            if (contains(path, localBrokerPath[0])) {
956                // Ignore this consumer as it's a consumer we locally sent to the broker.
957                LOG.debug("{} Ignoring sub from {}, already routed through this broker once: {}",
958                        configuration.getBrokerName(), remoteBrokerName, info);
959                return;
960            }
961
962            if (!isPermissableDestination(info.getDestination())) {
963                // ignore if not in the permitted or in the excluded list
964                LOG.debug("{} Ignoring sub from {}, destination {} is not permitted: {}",
965                        configuration.getBrokerName(), remoteBrokerName, info.getDestination(), info);
966                return;
967            }
968
969            // in a cyclic network there can be multiple bridges per broker that can propagate
970            // a network subscription so there is a need to synchronize on a shared entity
971            // if duplicate suppression is required
972            if (isDuplicateSuppressionOff(info)) {
973                addConsumerInfo(info);
974            } else {
975                synchronized (brokerService.getVmConnectorURI()) {
976                    addConsumerInfo(info);
977                }
978            }
979        } else if (data.getClass() == DestinationInfo.class) {
980            // It's a destination info - we want to pass up information about temporary destinations
981            final DestinationInfo destInfo = (DestinationInfo) data;
982            BrokerId[] path = destInfo.getBrokerPath();
983            if (path != null && networkTTL > -1 && path.length >= networkTTL) {
984                LOG.debug("{} Ignoring destination {} restricted to {} network hops only",
985                        configuration.getBrokerName(), destInfo, networkTTL);
986                return;
987            }
988            if (contains(destInfo.getBrokerPath(), localBrokerPath[0])) {
989                LOG.debug("{} Ignoring destination {} already routed through this broker once", configuration.getBrokerName(), destInfo);
990                return;
991            }
992            destInfo.setConnectionId(localConnectionInfo.getConnectionId());
993            if (destInfo.getDestination() instanceof ActiveMQTempDestination) {
994                // re-set connection id so comes from here
995                ActiveMQTempDestination tempDest = (ActiveMQTempDestination) destInfo.getDestination();
996                tempDest.setConnectionId(localSessionInfo.getSessionId().getConnectionId());
997            }
998            destInfo.setBrokerPath(appendToBrokerPath(destInfo.getBrokerPath(), getRemoteBrokerPath()));
999            LOG.trace("{} bridging {} destination on {} from {}, destination: {}",
1000                    configuration.getBrokerName(), (destInfo.isAddOperation() ? "add" : "remove"), localBroker, remoteBrokerName, destInfo);
1001            if (destInfo.isRemoveOperation()) {
1002                // Serialize with removeSub operations such that all removeSub advisories
1003                // are generated
1004                serialExecutor.execute(new Runnable() {
1005                    @Override
1006                    public void run() {
1007                        try {
1008                            localBroker.oneway(destInfo);
1009                        } catch (IOException e) {
1010                            LOG.warn("failed to deliver remove command for destination: {}", destInfo.getDestination(), e);
1011                        }
1012                    }
1013                });
1014            } else {
1015                localBroker.oneway(destInfo);
1016            }
1017        } else if (data.getClass() == RemoveInfo.class) {
1018            ConsumerId id = (ConsumerId) ((RemoveInfo) data).getObjectId();
1019            removeDemandSubscription(id);
1020
1021            if (forcedDurableRemoteId.remove(id)) {
1022                for (Iterator<DemandSubscription> i = subscriptionMapByLocalId.values().iterator(); i.hasNext(); ) {
1023                    DemandSubscription ds = i.next();
1024                    boolean removed = ds.removeForcedDurableConsumer(id);
1025                    if (removed) {
1026                        cleanupDurableSub(ds, i);
1027                    }
1028                }
1029           }
1030
1031        } else if (data.getClass() == RemoveSubscriptionInfo.class) {
1032            final RemoveSubscriptionInfo info = ((RemoveSubscriptionInfo) data);
1033            final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(info.getClientId(), info.getSubscriptionName());
1034            final boolean proxyBridgeSub = isProxyBridgeSubscription(subscriptionInfo.getClientId(),
1035                    subscriptionInfo.getSubscriptionName());
1036            for (Iterator<DemandSubscription> i = subscriptionMapByLocalId.values().iterator(); i.hasNext(); ) {
1037                DemandSubscription ds = i.next();
1038                boolean removed = ds.getDurableRemoteSubs().remove(subscriptionInfo);
1039
1040                //If this is a proxy bridge subscription we need to try changing the clientId
1041                if (!removed && proxyBridgeSub){
1042                    subscriptionInfo.setClientId(getProxyBridgeClientId(subscriptionInfo.getClientId()));
1043                    if (ds.getDurableRemoteSubs().contains(subscriptionInfo)) {
1044                        ds.getDurableRemoteSubs().remove(subscriptionInfo);
1045                        removed = true;
1046                    }
1047                }
1048
1049                if (removed) {
1050                    cleanupDurableSub(ds, i);
1051                }
1052            }
1053        }
1054    }
1055
1056    private void cleanupDurableSub(final DemandSubscription ds,
1057            Iterator<DemandSubscription> i) throws IOException {
1058
1059        if (ds != null && ds.getLocalDurableSubscriber() != null && ds.getDurableRemoteSubs().isEmpty()
1060                && ds.getForcedDurableConsumersSize() == 0) {
1061            // deactivate subscriber
1062            RemoveInfo removeInfo = new RemoveInfo(ds.getLocalInfo().getConsumerId());
1063            localBroker.oneway(removeInfo);
1064
1065            // remove subscriber
1066            RemoveSubscriptionInfo sending = new RemoveSubscriptionInfo();
1067            sending.setClientId(localClientId);
1068            sending.setSubscriptionName(ds.getLocalDurableSubscriber().getSubscriptionName());
1069            sending.setConnectionId(this.localConnectionInfo.getConnectionId());
1070            localBroker.oneway(sending);
1071
1072            //remove subscriber from local map
1073            i.remove();
1074
1075            //need to remove the mapping from the remote map as well
1076            subscriptionMapByRemoteId.remove(ds.getRemoteInfo().getConsumerId());
1077        }
1078    }
1079
1080    @Override
1081    public void serviceLocalException(Throwable error) {
1082        serviceLocalException(null, error);
1083    }
1084
1085    public void serviceLocalException(MessageDispatch messageDispatch, Throwable error) {
1086        LOG.trace("serviceLocalException: disposed {} ex", disposed.get(), error);
1087        if (!disposed.get()) {
1088            if (error instanceof DestinationDoesNotExistException && ((DestinationDoesNotExistException) error).isTemporary()) {
1089                // not a reason to terminate the bridge - temps can disappear with
1090                // pending sends as the demand sub may outlive the remote dest
1091                if (messageDispatch != null) {
1092                    LOG.warn("PoisonAck of {} on forwarding error: {}", messageDispatch.getMessage().getMessageId(), error);
1093                    try {
1094                        MessageAck poisonAck = new MessageAck(messageDispatch, MessageAck.POISON_ACK_TYPE, 1);
1095                        poisonAck.setPoisonCause(error);
1096                        localBroker.oneway(poisonAck);
1097                    } catch (IOException ioe) {
1098                        LOG.error("Failed to poison ack message following forward failure: ", ioe);
1099                    }
1100                    fireFailedForwardAdvisory(messageDispatch, error);
1101                } else {
1102                    LOG.warn("Ignoring exception on forwarding to non existent temp dest: ", error);
1103                }
1104                return;
1105            }
1106
1107            LOG.info("Network connection between {} and {} shutdown due to a local error: {}", localBroker, remoteBroker, error);
1108            LOG.debug("The local Exception was: {}", error, error);
1109
1110            brokerService.getTaskRunnerFactory().execute(new Runnable() {
1111                @Override
1112                public void run() {
1113                    ServiceSupport.dispose(getControllingService());
1114                }
1115            });
1116            fireBridgeFailed(error);
1117        }
1118    }
1119
1120    private void fireFailedForwardAdvisory(MessageDispatch messageDispatch, Throwable error) {
1121        if (configuration.isAdvisoryForFailedForward()) {
1122            AdvisoryBroker advisoryBroker = null;
1123            try {
1124                advisoryBroker = (AdvisoryBroker) brokerService.getBroker().getAdaptor(AdvisoryBroker.class);
1125
1126                if (advisoryBroker != null) {
1127                    ConnectionContext context = new ConnectionContext();
1128                    context.setSecurityContext(SecurityContext.BROKER_SECURITY_CONTEXT);
1129                    context.setBroker(brokerService.getBroker());
1130
1131                    ActiveMQMessage advisoryMessage = new ActiveMQMessage();
1132                    advisoryMessage.setStringProperty("cause", error.getLocalizedMessage());
1133                    advisoryBroker.fireAdvisory(context, AdvisorySupport.getNetworkBridgeForwardFailureAdvisoryTopic(), messageDispatch.getMessage(), null,
1134                            advisoryMessage);
1135
1136                }
1137            } catch (Exception e) {
1138                LOG.warn("failed to fire forward failure advisory, cause: {}", e);
1139                LOG.debug("detail", e);
1140            }
1141        }
1142    }
1143
1144    protected Service getControllingService() {
1145        return duplexInitiatingConnection != null ? duplexInitiatingConnection : DemandForwardingBridgeSupport.this;
1146    }
1147
1148    protected void addSubscription(DemandSubscription sub) throws IOException {
1149        if (sub != null) {
1150            localBroker.oneway(sub.getLocalInfo());
1151        }
1152    }
1153
1154    protected void removeSubscription(final DemandSubscription sub) throws IOException {
1155        if (sub != null) {
1156            LOG.trace("{} remove local subscription: {} for remote {}", configuration.getBrokerName(), sub.getLocalInfo().getConsumerId(), sub.getRemoteInfo().getConsumerId());
1157
1158            // ensure not available for conduit subs pending removal
1159            subscriptionMapByLocalId.remove(sub.getLocalInfo().getConsumerId());
1160            subscriptionMapByRemoteId.remove(sub.getRemoteInfo().getConsumerId());
1161
1162            // continue removal in separate thread to free up tshis thread for outstanding responses
1163            // Serialize with removeDestination operations so that removeSubs are serialized with
1164            // removeDestinations such that all removeSub advisories are generated
1165            serialExecutor.execute(new Runnable() {
1166                @Override
1167                public void run() {
1168                    sub.waitForCompletion();
1169                    try {
1170                        localBroker.oneway(sub.getLocalInfo().createRemoveCommand());
1171                    } catch (IOException e) {
1172                        LOG.warn("failed to deliver remove command for local subscription, for remote {}", sub.getRemoteInfo().getConsumerId(), e);
1173                    }
1174                }
1175            });
1176        }
1177    }
1178
1179    protected Message configureMessage(MessageDispatch md) throws IOException {
1180        Message message = md.getMessage().copy();
1181        // Update the packet to show where it came from.
1182        message.setBrokerPath(appendToBrokerPath(message.getBrokerPath(), localBrokerPath));
1183        message.setProducerId(producerInfo.getProducerId());
1184        message.setDestination(md.getDestination());
1185        message.setMemoryUsage(null);
1186        if (message.getOriginalTransactionId() == null) {
1187            message.setOriginalTransactionId(message.getTransactionId());
1188        }
1189        message.setTransactionId(null);
1190        if (configuration.isUseCompression()) {
1191            message.compress();
1192        }
1193        return message;
1194    }
1195
1196    protected void serviceLocalCommand(Command command) {
1197        if (!disposed.get()) {
1198            try {
1199                if (command.isMessageDispatch()) {
1200                    safeWaitUntilStarted();
1201                    networkBridgeStatistics.getEnqueues().increment();
1202                    final MessageDispatch md = (MessageDispatch) command;
1203                    final DemandSubscription sub = subscriptionMapByLocalId.get(md.getConsumerId());
1204                    if (sub != null && md.getMessage() != null && sub.incrementOutstandingResponses()) {
1205
1206                        if (suppressMessageDispatch(md, sub)) {
1207                            LOG.debug("{} message not forwarded to {} because message came from there or fails TTL, brokerPath: {}, message: {}",
1208                                    configuration.getBrokerName(), remoteBrokerName, Arrays.toString(md.getMessage().getBrokerPath()), md.getMessage());
1209                            // still ack as it may be durable
1210                            try {
1211                                localBroker.oneway(new MessageAck(md, MessageAck.INDIVIDUAL_ACK_TYPE, 1));
1212                            } finally {
1213                                sub.decrementOutstandingResponses();
1214                            }
1215                            return;
1216                        }
1217
1218                        Message message = configureMessage(md);
1219                        LOG.debug("bridging ({} -> {}), consumer: {}, destination: {}, brokerPath: {}, message: {}",
1220                                configuration.getBrokerName(), remoteBrokerName, md.getConsumerId(), message.getDestination(), Arrays.toString(message.getBrokerPath()), (LOG.isTraceEnabled() ? message : message.getMessageId()));
1221                        if (isDuplex() && NetworkBridgeFilter.isAdvisoryInterpretedByNetworkBridge(message)) {
1222                            try {
1223                                // never request b/c they are eventually                     acked async
1224                                remoteBroker.oneway(message);
1225                            } finally {
1226                                sub.decrementOutstandingResponses();
1227                            }
1228                            return;
1229                        }
1230                        if (isPermissableDestination(md.getDestination())) {
1231                           if (message.isPersistent() || configuration.isAlwaysSyncSend()) {
1232
1233                              // The message was not sent using async send, so we should only
1234                              // ack the local broker when we get confirmation that the remote
1235                              // broker has received the message.
1236                              remoteBroker.asyncRequest(message, new ResponseCallback() {
1237                                 @Override
1238                                 public void onCompletion(FutureResponse future) {
1239                                    try {
1240                                       Response response = future.getResult();
1241                                       if (response.isException()) {
1242                                          ExceptionResponse er = (ExceptionResponse) response;
1243                                          serviceLocalException(md, er.getException());
1244                                       } else {
1245                                          localBroker.oneway(new MessageAck(md, MessageAck.INDIVIDUAL_ACK_TYPE, 1));
1246                                          networkBridgeStatistics.getDequeues().increment();
1247                                       }
1248                                    } catch (IOException e) {
1249                                       serviceLocalException(md, e);
1250                                    } finally {
1251                                       sub.decrementOutstandingResponses();
1252                                    }
1253                                 }
1254                              });
1255
1256                           } else {
1257                              // If the message was originally sent using async send, we will
1258                              // preserve that QOS by bridging it using an async send (small chance
1259                              // of message loss).
1260                              try {
1261                                 remoteBroker.oneway(message);
1262                                 localBroker.oneway(new MessageAck(md, MessageAck.INDIVIDUAL_ACK_TYPE, 1));
1263                                 networkBridgeStatistics.getDequeues().increment();
1264                              } finally {
1265                                 sub.decrementOutstandingResponses();
1266                              }
1267                           }
1268                           serviceOutbound(message);
1269                        }
1270                    } else {
1271                        LOG.debug("No subscription registered with this network bridge for consumerId: {} for message: {}", md.getConsumerId(), md.getMessage());
1272                    }
1273                } else if (command.isBrokerInfo()) {
1274                    futureLocalBrokerInfo.set((BrokerInfo) command);
1275                } else if (command.isShutdownInfo()) {
1276                    LOG.info("{} Shutting down {}", configuration.getBrokerName(), configuration.getName());
1277                    stop();
1278                } else if (command.getClass() == ConnectionError.class) {
1279                    ConnectionError ce = (ConnectionError) command;
1280                    serviceLocalException(ce.getException());
1281                } else {
1282                    switch (command.getDataStructureType()) {
1283                        case WireFormatInfo.DATA_STRUCTURE_TYPE:
1284                            break;
1285                        case BrokerSubscriptionInfo.DATA_STRUCTURE_TYPE:
1286                            break;
1287                        default:
1288                            LOG.warn("Unexpected local command: {}", command);
1289                    }
1290                }
1291            } catch (Throwable e) {
1292                LOG.warn("Caught an exception processing local command", e);
1293                serviceLocalException(e);
1294            }
1295        }
1296    }
1297
1298    private boolean suppressMessageDispatch(MessageDispatch md, DemandSubscription sub) throws Exception {
1299        boolean suppress = false;
1300        // for durable subs, suppression via filter leaves dangling acks so we
1301        // need to check here and allow the ack irrespective
1302        if (sub.getLocalInfo().isDurable()) {
1303            NonCachedMessageEvaluationContext messageEvalContext = new NonCachedMessageEvaluationContext();
1304            messageEvalContext.setMessageReference(md.getMessage());
1305            messageEvalContext.setDestination(md.getDestination());
1306            suppress = !sub.getNetworkBridgeFilter().matches(messageEvalContext);
1307        }
1308        return suppress;
1309    }
1310
1311    public static boolean contains(BrokerId[] brokerPath, BrokerId brokerId) {
1312        if (brokerPath != null) {
1313            for (BrokerId id : brokerPath) {
1314                if (brokerId.equals(id)) {
1315                    return true;
1316                }
1317            }
1318        }
1319        return false;
1320    }
1321
1322    protected BrokerId[] appendToBrokerPath(BrokerId[] brokerPath, BrokerId[] pathsToAppend) {
1323        if (brokerPath == null || brokerPath.length == 0) {
1324            return pathsToAppend;
1325        }
1326        BrokerId rc[] = new BrokerId[brokerPath.length + pathsToAppend.length];
1327        System.arraycopy(brokerPath, 0, rc, 0, brokerPath.length);
1328        System.arraycopy(pathsToAppend, 0, rc, brokerPath.length, pathsToAppend.length);
1329        return rc;
1330    }
1331
1332    protected BrokerId[] appendToBrokerPath(BrokerId[] brokerPath, BrokerId idToAppend) {
1333        if (brokerPath == null || brokerPath.length == 0) {
1334            return new BrokerId[]{idToAppend};
1335        }
1336        BrokerId rc[] = new BrokerId[brokerPath.length + 1];
1337        System.arraycopy(brokerPath, 0, rc, 0, brokerPath.length);
1338        rc[brokerPath.length] = idToAppend;
1339        return rc;
1340    }
1341
1342    protected boolean isPermissableDestination(ActiveMQDestination destination) {
1343        return isPermissableDestination(destination, false);
1344    }
1345
1346    protected boolean isPermissableDestination(ActiveMQDestination destination, boolean allowTemporary) {
1347        // Are we not bridging temporary destinations?
1348        if (destination.isTemporary()) {
1349            if (allowTemporary) {
1350                return true;
1351            } else {
1352                return configuration.isBridgeTempDestinations();
1353            }
1354        }
1355
1356        ActiveMQDestination[] dests = excludedDestinations;
1357        if (dests != null && dests.length > 0) {
1358            for (ActiveMQDestination dest : dests) {
1359                DestinationFilter exclusionFilter = DestinationFilter.parseFilter(dest);
1360                if (dest != null && exclusionFilter.matches(destination) && dest.getDestinationType() == destination.getDestinationType()) {
1361                    return false;
1362                }
1363            }
1364        }
1365
1366        dests = staticallyIncludedDestinations;
1367        if (dests != null && dests.length > 0) {
1368            for (ActiveMQDestination dest : dests) {
1369                DestinationFilter inclusionFilter = DestinationFilter.parseFilter(dest);
1370                if (dest != null && inclusionFilter.matches(destination) && dest.getDestinationType() == destination.getDestinationType()) {
1371                    return true;
1372                }
1373            }
1374        }
1375
1376        dests = dynamicallyIncludedDestinations;
1377        if (dests != null && dests.length > 0) {
1378            for (ActiveMQDestination dest : dests) {
1379                DestinationFilter inclusionFilter = DestinationFilter.parseFilter(dest);
1380                if (dest != null && inclusionFilter.matches(destination) && dest.getDestinationType() == destination.getDestinationType()) {
1381                    return true;
1382                }
1383            }
1384
1385            return false;
1386        }
1387
1388        return true;
1389    }
1390
1391    /**
1392     * Subscriptions for these destinations are always created
1393     */
1394    protected void setupStaticDestinations() {
1395        ActiveMQDestination[] dests = staticallyIncludedDestinations;
1396        if (dests != null) {
1397            for (ActiveMQDestination dest : dests) {
1398                if (isPermissableDestination(dest)) {
1399                    DemandSubscription sub = createDemandSubscription(dest, null);
1400                    if (sub != null) {
1401                        sub.setStaticallyIncluded(true);
1402                        try {
1403                            addSubscription(sub);
1404                        } catch (IOException e) {
1405                            LOG.error("Failed to add static destination {}", dest, e);
1406                        }
1407                        LOG.trace("{}, bridging messages for static destination: {}", configuration.getBrokerName(), dest);
1408                    } else {
1409                        LOG.info("{}, static destination excluded: {}, demand already exists", configuration.getBrokerName(), dest);
1410                    }
1411                } else {
1412                    LOG.info("{}, static destination excluded: {}", configuration.getBrokerName(), dest);
1413                }
1414            }
1415        }
1416    }
1417
1418    protected void addConsumerInfo(final ConsumerInfo consumerInfo) throws IOException {
1419        ConsumerInfo info = consumerInfo.copy();
1420        addRemoteBrokerToBrokerPath(info);
1421        DemandSubscription sub = createDemandSubscription(info);
1422        if (sub != null) {
1423            if (duplicateSuppressionIsRequired(sub)) {
1424                undoMapRegistration(sub);
1425            } else {
1426                if (consumerInfo.isDurable()) {
1427                    //Handle the demand generated by proxy network subscriptions
1428                    //The broker path is case is normal
1429                    if (isProxyNSConsumerBrokerPath(sub.getRemoteInfo()) &&
1430                            info.getSubscriptionName() != null && info.getSubscriptionName().startsWith(DURABLE_SUB_PREFIX)) {
1431                        final BrokerId[] path = info.getBrokerPath();
1432                        addProxyNetworkSubscriptionBrokerPath(sub, path, consumerInfo.getSubscriptionName());
1433                    //This is the durable sync case on broker restart
1434                    } else if (isProxyNSConsumerClientId(sub.getRemoteInfo().getClientId()) &&
1435                            isProxyBridgeSubscription(info.getClientId(), info.getSubscriptionName())) {
1436                                addProxyNetworkSubscriptionClientId(sub, sub.getRemoteInfo().getClientId(), consumerInfo.getSubscriptionName());
1437                        } else {
1438                                sub.getDurableRemoteSubs().add(new SubscriptionInfo(sub.getRemoteInfo().getClientId(), consumerInfo.getSubscriptionName()));
1439                        }
1440                }
1441                addSubscription(sub);
1442                LOG.debug("{} new demand subscription: {}", configuration.getBrokerName(), sub);
1443            }
1444        }
1445    }
1446
1447    private void undoMapRegistration(DemandSubscription sub) {
1448        subscriptionMapByLocalId.remove(sub.getLocalInfo().getConsumerId());
1449        subscriptionMapByRemoteId.remove(sub.getRemoteInfo().getConsumerId());
1450    }
1451
1452    /*
1453     * check our existing subs networkConsumerIds against the list of network
1454     * ids in this subscription A match means a duplicate which we suppress for
1455     * topics and maybe for queues
1456     */
1457    private boolean duplicateSuppressionIsRequired(DemandSubscription candidate) {
1458        final ConsumerInfo consumerInfo = candidate.getRemoteInfo();
1459        boolean suppress = false;
1460
1461        if (isDuplicateSuppressionOff(consumerInfo)) {
1462            return suppress;
1463        }
1464
1465        List<ConsumerId> candidateConsumers = consumerInfo.getNetworkConsumerIds();
1466        Collection<Subscription> currentSubs = getRegionSubscriptions(consumerInfo.getDestination());
1467        for (Subscription sub : currentSubs) {
1468            List<ConsumerId> networkConsumers = sub.getConsumerInfo().getNetworkConsumerIds();
1469            if (!networkConsumers.isEmpty()) {
1470                if (matchFound(candidateConsumers, networkConsumers)) {
1471                    if (isInActiveDurableSub(sub)) {
1472                        suppress = false;
1473                    } else {
1474                        suppress = hasLowerPriority(sub, candidate.getLocalInfo());
1475                    }
1476                    break;
1477                }
1478            }
1479        }
1480        return suppress;
1481    }
1482
1483    private boolean isDuplicateSuppressionOff(final ConsumerInfo consumerInfo) {
1484        return !configuration.isSuppressDuplicateQueueSubscriptions() && !configuration.isSuppressDuplicateTopicSubscriptions()
1485                || consumerInfo.getDestination().isQueue() && !configuration.isSuppressDuplicateQueueSubscriptions()
1486                || consumerInfo.getDestination().isTopic() && !configuration.isSuppressDuplicateTopicSubscriptions();
1487    }
1488
1489    private boolean isInActiveDurableSub(Subscription sub) {
1490        return (sub.getConsumerInfo().isDurable() && sub instanceof DurableTopicSubscription && !((DurableTopicSubscription) sub).isActive());
1491    }
1492
1493    private boolean hasLowerPriority(Subscription existingSub, ConsumerInfo candidateInfo) {
1494        boolean suppress = false;
1495
1496        if (existingSub.getConsumerInfo().getPriority() >= candidateInfo.getPriority()) {
1497            LOG.debug("{} Ignoring duplicate subscription from {}, sub: {} is duplicate by network subscription with equal or higher network priority: {}, networkConsumerIds: {}",
1498                    configuration.getBrokerName(), remoteBrokerName, candidateInfo, existingSub, existingSub.getConsumerInfo().getNetworkConsumerIds());
1499            suppress = true;
1500        } else {
1501            // remove the existing lower priority duplicate and allow this candidate
1502            try {
1503                removeDuplicateSubscription(existingSub);
1504
1505                LOG.debug("{} Replacing duplicate subscription {} with sub from {}, which has a higher priority, new sub: {}, networkConsumerIds: {}",
1506                        configuration.getBrokerName(), existingSub.getConsumerInfo(), remoteBrokerName, candidateInfo, candidateInfo.getNetworkConsumerIds());
1507            } catch (IOException e) {
1508                LOG.error("Failed to remove duplicated sub as a result of sub with higher priority, sub: {}", existingSub, e);
1509            }
1510        }
1511        return suppress;
1512    }
1513
1514    private void removeDuplicateSubscription(Subscription existingSub) throws IOException {
1515        for (NetworkConnector connector : brokerService.getNetworkConnectors()) {
1516            if (connector.removeDemandSubscription(existingSub.getConsumerInfo().getConsumerId())) {
1517                break;
1518            }
1519        }
1520    }
1521
1522    private boolean matchFound(List<ConsumerId> candidateConsumers, List<ConsumerId> networkConsumers) {
1523        boolean found = false;
1524        for (ConsumerId aliasConsumer : networkConsumers) {
1525            if (candidateConsumers.contains(aliasConsumer)) {
1526                found = true;
1527                break;
1528            }
1529        }
1530        return found;
1531    }
1532
1533    protected final Collection<Subscription> getRegionSubscriptions(ActiveMQDestination dest) {
1534        RegionBroker region_broker = (RegionBroker) brokerService.getRegionBroker();
1535        Region region;
1536        Collection<Subscription> subs;
1537
1538        region = null;
1539        switch (dest.getDestinationType()) {
1540            case ActiveMQDestination.QUEUE_TYPE:
1541                region = region_broker.getQueueRegion();
1542                break;
1543            case ActiveMQDestination.TOPIC_TYPE:
1544                region = region_broker.getTopicRegion();
1545                break;
1546            case ActiveMQDestination.TEMP_QUEUE_TYPE:
1547                region = region_broker.getTempQueueRegion();
1548                break;
1549            case ActiveMQDestination.TEMP_TOPIC_TYPE:
1550                region = region_broker.getTempTopicRegion();
1551                break;
1552        }
1553
1554        if (region instanceof AbstractRegion) {
1555            subs = ((AbstractRegion) region).getSubscriptions().values();
1556        } else {
1557            subs = null;
1558        }
1559
1560        return subs;
1561    }
1562
1563    protected DemandSubscription createDemandSubscription(ConsumerInfo info) throws IOException {
1564        // add our original id to ourselves
1565        info.addNetworkConsumerId(info.getConsumerId());
1566        return doCreateDemandSubscription(info);
1567    }
1568
1569    protected DemandSubscription doCreateDemandSubscription(ConsumerInfo info) throws IOException {
1570        DemandSubscription result = new DemandSubscription(info);
1571        result.getLocalInfo().setConsumerId(new ConsumerId(localSessionInfo.getSessionId(), consumerIdGenerator.getNextSequenceId()));
1572        if (info.getDestination().isTemporary()) {
1573            // reset the local connection Id
1574            ActiveMQTempDestination dest = (ActiveMQTempDestination) result.getLocalInfo().getDestination();
1575            dest.setConnectionId(localConnectionInfo.getConnectionId().toString());
1576        }
1577
1578        if (configuration.isDecreaseNetworkConsumerPriority()) {
1579            byte priority = (byte) configuration.getConsumerPriorityBase();
1580            if (info.getBrokerPath() != null && info.getBrokerPath().length > 1) {
1581                // The longer the path to the consumer, the less it's consumer priority.
1582                priority -= info.getBrokerPath().length + 1;
1583            }
1584            result.getLocalInfo().setPriority(priority);
1585            LOG.debug("{} using priority: {} for subscription: {}", configuration.getBrokerName(), priority, info);
1586        }
1587        configureDemandSubscription(info, result);
1588        return result;
1589    }
1590
1591    final protected DemandSubscription createDemandSubscription(ActiveMQDestination destination, final String subscriptionName) {
1592        ConsumerInfo info = new ConsumerInfo();
1593        info.setNetworkSubscription(true);
1594        info.setDestination(destination);
1595
1596        if (subscriptionName != null) {
1597            info.setSubscriptionName(subscriptionName);
1598        }
1599
1600        // Indicate that this subscription is being made on behalf of the remote broker.
1601        info.setBrokerPath(new BrokerId[]{remoteBrokerId});
1602
1603        // the remote info held by the DemandSubscription holds the original
1604        // consumerId, the local info get's overwritten
1605        info.setConsumerId(new ConsumerId(localSessionInfo.getSessionId(), consumerIdGenerator.getNextSequenceId()));
1606        DemandSubscription result = null;
1607        try {
1608            result = createDemandSubscription(info);
1609        } catch (IOException e) {
1610            LOG.error("Failed to create DemandSubscription ", e);
1611        }
1612        return result;
1613    }
1614
1615    protected void configureDemandSubscription(ConsumerInfo info, DemandSubscription sub) throws IOException {
1616        if (AdvisorySupport.isConsumerAdvisoryTopic(info.getDestination()) ||
1617                AdvisorySupport.isVirtualDestinationConsumerAdvisoryTopic(info.getDestination())) {
1618            sub.getLocalInfo().setDispatchAsync(true);
1619        } else {
1620            sub.getLocalInfo().setDispatchAsync(configuration.isDispatchAsync());
1621        }
1622        configureConsumerPrefetch(sub.getLocalInfo());
1623        subscriptionMapByLocalId.put(sub.getLocalInfo().getConsumerId(), sub);
1624        subscriptionMapByRemoteId.put(sub.getRemoteInfo().getConsumerId(), sub);
1625
1626        sub.setNetworkBridgeFilter(createNetworkBridgeFilter(info));
1627        if (!info.isDurable()) {
1628            // This works for now since we use a VM connection to the local broker.
1629            // may need to change if we ever subscribe to a remote broker.
1630            sub.getLocalInfo().setAdditionalPredicate(sub.getNetworkBridgeFilter());
1631        } else {
1632            sub.setLocalDurableSubscriber(new SubscriptionInfo(info.getClientId(), info.getSubscriptionName()));
1633        }
1634    }
1635
1636    protected void removeDemandSubscription(ConsumerId id) throws IOException {
1637        DemandSubscription sub = subscriptionMapByRemoteId.remove(id);
1638        LOG.debug("{} remove request on {} from {}, consumer id: {}, matching sub: {}",
1639                configuration.getBrokerName(), localBroker, remoteBrokerName, id, sub);
1640        if (sub != null) {
1641            removeSubscription(sub);
1642            LOG.debug("{} removed sub on {} from {}: {}",
1643                    configuration.getBrokerName(), localBroker, remoteBrokerName, sub.getRemoteInfo());
1644        }
1645    }
1646
1647    protected boolean removeDemandSubscriptionByLocalId(ConsumerId consumerId) {
1648        boolean removeDone = false;
1649        DemandSubscription sub = subscriptionMapByLocalId.get(consumerId);
1650        if (sub != null) {
1651            try {
1652                removeDemandSubscription(sub.getRemoteInfo().getConsumerId());
1653                removeDone = true;
1654            } catch (IOException e) {
1655                LOG.debug("removeDemandSubscriptionByLocalId failed for localId: {}", consumerId, e);
1656            }
1657        }
1658        return removeDone;
1659    }
1660
1661    /**
1662     * Performs a timed wait on the started latch and then checks for disposed
1663     * before performing another wait each time the the started wait times out.
1664     */
1665    protected boolean safeWaitUntilStarted() throws InterruptedException {
1666        while (!disposed.get()) {
1667            if (startedLatch.await(1, TimeUnit.SECONDS)) {
1668                break;
1669            }
1670        }
1671        return !disposed.get();
1672    }
1673
1674    protected NetworkBridgeFilter createNetworkBridgeFilter(ConsumerInfo info) throws IOException {
1675        NetworkBridgeFilterFactory filterFactory = defaultFilterFactory;
1676        if (brokerService != null && brokerService.getDestinationPolicy() != null) {
1677            PolicyEntry entry = brokerService.getDestinationPolicy().getEntryFor(info.getDestination());
1678            if (entry != null && entry.getNetworkBridgeFilterFactory() != null) {
1679                filterFactory = entry.getNetworkBridgeFilterFactory();
1680            }
1681        }
1682        return filterFactory.create(info, getRemoteBrokerPath(), configuration.getMessageTTL(), configuration.getConsumerTTL());
1683    }
1684
1685    protected void addRemoteBrokerToBrokerPath(ConsumerInfo info) throws IOException {
1686        info.setBrokerPath(appendToBrokerPath(info.getBrokerPath(), getRemoteBrokerPath()));
1687    }
1688
1689    protected BrokerId[] getRemoteBrokerPath() {
1690        return remoteBrokerPath;
1691    }
1692
1693    @Override
1694    public void setNetworkBridgeListener(NetworkBridgeListener listener) {
1695        this.networkBridgeListener = listener;
1696    }
1697
1698    private void fireBridgeFailed(Throwable reason) {
1699        LOG.trace("fire bridge failed, listener: {}", this.networkBridgeListener, reason);
1700        NetworkBridgeListener l = this.networkBridgeListener;
1701        if (l != null && this.bridgeFailed.compareAndSet(false, true)) {
1702            l.bridgeFailed();
1703        }
1704    }
1705
1706    /**
1707     * @return Returns the dynamicallyIncludedDestinations.
1708     */
1709    public ActiveMQDestination[] getDynamicallyIncludedDestinations() {
1710        return dynamicallyIncludedDestinations;
1711    }
1712
1713    /**
1714     * @param dynamicallyIncludedDestinations
1715     *         The dynamicallyIncludedDestinations to set.
1716     */
1717    public void setDynamicallyIncludedDestinations(ActiveMQDestination[] dynamicallyIncludedDestinations) {
1718        this.dynamicallyIncludedDestinations = dynamicallyIncludedDestinations;
1719    }
1720
1721    /**
1722     * @return Returns the excludedDestinations.
1723     */
1724    public ActiveMQDestination[] getExcludedDestinations() {
1725        return excludedDestinations;
1726    }
1727
1728    /**
1729     * @param excludedDestinations The excludedDestinations to set.
1730     */
1731    public void setExcludedDestinations(ActiveMQDestination[] excludedDestinations) {
1732        this.excludedDestinations = excludedDestinations;
1733    }
1734
1735    /**
1736     * @return Returns the staticallyIncludedDestinations.
1737     */
1738    public ActiveMQDestination[] getStaticallyIncludedDestinations() {
1739        return staticallyIncludedDestinations;
1740    }
1741
1742    /**
1743     * @param staticallyIncludedDestinations The staticallyIncludedDestinations to set.
1744     */
1745    public void setStaticallyIncludedDestinations(ActiveMQDestination[] staticallyIncludedDestinations) {
1746        this.staticallyIncludedDestinations = staticallyIncludedDestinations;
1747    }
1748
1749    /**
1750     * @return Returns the durableDestinations.
1751     */
1752    public ActiveMQDestination[] getDurableDestinations() {
1753        return durableDestinations;
1754    }
1755
1756    /**
1757     * @param durableDestinations The durableDestinations to set.
1758     */
1759    public void setDurableDestinations(ActiveMQDestination[] durableDestinations) {
1760        this.durableDestinations = durableDestinations;
1761    }
1762
1763    /**
1764     * @return Returns the localBroker.
1765     */
1766    public Transport getLocalBroker() {
1767        return localBroker;
1768    }
1769
1770    /**
1771     * @return Returns the remoteBroker.
1772     */
1773    public Transport getRemoteBroker() {
1774        return remoteBroker;
1775    }
1776
1777    /**
1778     * @return the createdByDuplex
1779     */
1780    public boolean isCreatedByDuplex() {
1781        return this.createdByDuplex;
1782    }
1783
1784    /**
1785     * @param createdByDuplex the createdByDuplex to set
1786     */
1787    public void setCreatedByDuplex(boolean createdByDuplex) {
1788        this.createdByDuplex = createdByDuplex;
1789    }
1790
1791    @Override
1792    public String getRemoteAddress() {
1793        return remoteBroker.getRemoteAddress();
1794    }
1795
1796    @Override
1797    public String getLocalAddress() {
1798        return localBroker.getRemoteAddress();
1799    }
1800
1801    @Override
1802    public String getRemoteBrokerName() {
1803        return remoteBrokerInfo == null ? null : remoteBrokerInfo.getBrokerName();
1804    }
1805
1806    @Override
1807    public String getRemoteBrokerId() {
1808        return (remoteBrokerInfo == null || remoteBrokerInfo.getBrokerId() == null) ? null : remoteBrokerInfo.getBrokerId().toString();
1809    }
1810
1811    @Override
1812    public String getLocalBrokerName() {
1813        return localBrokerInfo == null ? null : localBrokerInfo.getBrokerName();
1814    }
1815
1816    @Override
1817    public long getDequeueCounter() {
1818        return networkBridgeStatistics.getDequeues().getCount();
1819    }
1820
1821    @Override
1822    public long getEnqueueCounter() {
1823        return networkBridgeStatistics.getEnqueues().getCount();
1824    }
1825
1826    @Override
1827    public NetworkBridgeStatistics getNetworkBridgeStatistics() {
1828        return networkBridgeStatistics;
1829    }
1830
1831    protected boolean isDuplex() {
1832        return configuration.isDuplex() || createdByDuplex;
1833    }
1834
1835    public ConcurrentMap<ConsumerId, DemandSubscription> getLocalSubscriptionMap() {
1836        return subscriptionMapByRemoteId;
1837    }
1838
1839    @Override
1840    public void setBrokerService(BrokerService brokerService) {
1841        this.brokerService = brokerService;
1842        this.localBrokerId = brokerService.getRegionBroker().getBrokerId();
1843        localBrokerPath[0] = localBrokerId;
1844    }
1845
1846    @Override
1847    public void setMbeanObjectName(ObjectName objectName) {
1848        this.mbeanObjectName = objectName;
1849    }
1850
1851    @Override
1852    public ObjectName getMbeanObjectName() {
1853        return mbeanObjectName;
1854    }
1855
1856    @Override
1857    public void resetStats() {
1858        networkBridgeStatistics.reset();
1859    }
1860
1861    /*
1862     * Used to allow for async tasks to await receipt of the BrokerInfo from the local and
1863     * remote sides of the network bridge.
1864     */
1865    private static class FutureBrokerInfo implements Future<BrokerInfo> {
1866
1867        private final CountDownLatch slot = new CountDownLatch(1);
1868        private final AtomicBoolean disposed;
1869        private volatile BrokerInfo info = null;
1870
1871        public FutureBrokerInfo(BrokerInfo info, AtomicBoolean disposed) {
1872            this.info = info;
1873            this.disposed = disposed;
1874        }
1875
1876        @Override
1877        public boolean cancel(boolean mayInterruptIfRunning) {
1878            slot.countDown();
1879            return true;
1880        }
1881
1882        @Override
1883        public boolean isCancelled() {
1884            return slot.getCount() == 0 && info == null;
1885        }
1886
1887        @Override
1888        public boolean isDone() {
1889            return info != null;
1890        }
1891
1892        @Override
1893        public BrokerInfo get() throws InterruptedException, ExecutionException {
1894            try {
1895                if (info == null) {
1896                    while (!disposed.get()) {
1897                        if (slot.await(1, TimeUnit.SECONDS)) {
1898                            break;
1899                        }
1900                    }
1901                }
1902                return info;
1903            } catch (InterruptedException e) {
1904                Thread.currentThread().interrupt();
1905                LOG.debug("Operation interrupted: {}", e, e);
1906                throw new InterruptedException("Interrupted.");
1907            }
1908        }
1909
1910        @Override
1911        public BrokerInfo get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
1912            try {
1913                if (info == null) {
1914                    long deadline = System.currentTimeMillis() + unit.toMillis(timeout);
1915
1916                    while (!disposed.get() || System.currentTimeMillis() - deadline < 0) {
1917                        if (slot.await(1, TimeUnit.MILLISECONDS)) {
1918                            break;
1919                        }
1920                    }
1921                    if (info == null) {
1922                        throw new TimeoutException();
1923                    }
1924                }
1925                return info;
1926            } catch (InterruptedException e) {
1927                throw new InterruptedException("Interrupted.");
1928            }
1929        }
1930
1931        public void set(BrokerInfo info) {
1932            this.info = info;
1933            this.slot.countDown();
1934        }
1935    }
1936
1937    protected void serviceOutbound(Message message) {
1938        NetworkBridgeListener l = this.networkBridgeListener;
1939        if (l != null) {
1940            l.onOutboundMessage(this, message);
1941        }
1942    }
1943
1944    protected void serviceInboundMessage(Message message) {
1945        NetworkBridgeListener l = this.networkBridgeListener;
1946        if (l != null) {
1947            l.onInboundMessage(this, message);
1948        }
1949    }
1950
1951    protected boolean canDuplexDispatch(Message message) {
1952        boolean result = true;
1953        if (configuration.isCheckDuplicateMessagesOnDuplex()){
1954            final long producerSequenceId = message.getMessageId().getProducerSequenceId();
1955            //  messages are multiplexed on this producer so we need to query the persistenceAdapter
1956            long lastStoredForMessageProducer = getStoredSequenceIdForMessage(message.getMessageId());
1957            if (producerSequenceId <= lastStoredForMessageProducer) {
1958                result = false;
1959                LOG.debug("suppressing duplicate message send [{}] from network producer with producerSequence [{}] less than last stored: {}",
1960                        (LOG.isTraceEnabled() ? message : message.getMessageId()), producerSequenceId, lastStoredForMessageProducer);
1961            }
1962        }
1963        return result;
1964    }
1965
1966    protected long getStoredSequenceIdForMessage(MessageId messageId) {
1967        try {
1968            return brokerService.getPersistenceAdapter().getLastProducerSequenceId(messageId.getProducerId());
1969        } catch (IOException ignored) {
1970            LOG.debug("Failed to determine last producer sequence id for: {}", messageId, ignored);
1971        }
1972        return -1;
1973    }
1974
1975    protected void configureConsumerPrefetch(ConsumerInfo consumerInfo) {
1976        //If a consumer on an advisory topic and advisoryPrefetchSize has been explicitly
1977        //set then use it, else default to the prefetchSize setting
1978        if (AdvisorySupport.isAdvisoryTopic(consumerInfo.getDestination()) &&
1979                configuration.getAdvisoryPrefetchSize() > 0) {
1980            consumerInfo.setPrefetchSize(configuration.getAdvisoryPrefetchSize());
1981        } else {
1982            consumerInfo.setPrefetchSize(configuration.getPrefetchSize());
1983        }
1984    }
1985
1986}