/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.activemq.transport.failover;

import org.apache.activemq.broker.TransportConnector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Complex cluster test that will exercise the dynamic failover capabilities of
 * a network of brokers. Using a networking of 3 brokers where the 3rd broker is
 * removed and then added back in it is expected in each test that the number of
 * connections on the client should start with 3, then have two after the 3rd
 * broker is removed and then show 3 after the 3rd broker is reintroduced.
 */
public class FailoverComplexClusterTest extends FailoverClusterTestSupport {

   private static final String BROKER_A_CLIENT_TC_ADDRESS = "tcp://127.0.0.1:61616";
   private static final String BROKER_B_CLIENT_TC_ADDRESS = "tcp://127.0.0.1:61617";
   private static final String BROKER_C_CLIENT_TC_ADDRESS = "tcp://127.0.0.1:61618";
   private static final String BROKER_A_NOB_TC_ADDRESS = "tcp://127.0.0.1:61626";
   private static final String BROKER_B_NOB_TC_ADDRESS = "tcp://127.0.0.1:61627";
   private static final String BROKER_C_NOB_TC_ADDRESS = "tcp://127.0.0.1:61628";
   private static final String BROKER_A_NAME = "BROKERA";
   private static final String BROKER_B_NAME = "BROKERB";
   private static final String BROKER_C_NAME = "BROKERC";

   /**
    * Basic dynamic failover 3 broker test
    *
    * @throws Exception
    */
   public void testThreeBrokerClusterSingleConnectorBasic() throws Exception {

      initSingleTcBroker("", null, null);

      Thread.sleep(2000);

      setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
      createClients();
      Thread.sleep(2000);

      runTests(false, null, null, null);
   }

   /**
    * Tests a 3 broker configuration to ensure that the backup is random and
    * supported in a cluster. useExponentialBackOff is set to false and
    * maxReconnectAttempts is set to 1 to move through the list quickly for
    * this test.
    *
    * @throws Exception
    */
   public void testThreeBrokerClusterSingleConnectorBackupFailoverConfig() throws Exception {

      initSingleTcBroker("", null, null);

      Thread.sleep(2000);

      setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")?backup=true&backupPoolSize=2&useExponentialBackOff=false&initialReconnectDelay=500");
      createClients();
      Thread.sleep(2000);

      runTests(false, null, null, null);
   }

   /**
    * Tests a 3 broker cluster that passes in connection params on the
    * transport connector. Prior versions of AMQ passed the TC connection
    * params to the client and this should not happen. The chosen param is not
    * compatible with the client and will throw an error if used.
    *
    * @throws Exception
    */
   public void testThreeBrokerClusterSingleConnectorWithParams() throws Exception {

      initSingleTcBroker("?transport.closeAsync=false", null, null);

      Thread.sleep(2000);
      setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
      createClients();
      Thread.sleep(2000);

      runTests(false, null, null, null);
   }

   /**
    * Tests a 3 broker cluster using a cluster filter of *
    *
    * @throws Exception
    */
   public void testThreeBrokerClusterWithClusterFilter() throws Exception {

      initSingleTcBroker("?transport.closeAsync=false", null, null);

      Thread.sleep(2000);
      setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
      createClients();

      runTests(false, null, "*", null);
   }

   /**
    * Test to verify that a broker with multiple transport connections only the
    * one marked to update clients is propagate
    *
    * @throws Exception
    */
   public void testThreeBrokerClusterMultipleConnectorBasic() throws Exception {

      initMultiTcCluster("", null);

      Thread.sleep(2000);

      setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
      createClients();
      Thread.sleep(2000);

      runTests(true, null, null, null);
   }

   /**
    * Test to verify the reintroduction of the A Broker
    *
    * @throws Exception
    */
   public void testOriginalBrokerRestart() throws Exception {
      initSingleTcBroker("", null, null);

      Thread.sleep(2000);

      setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
      createClients();
      Thread.sleep(2000);

      assertClientsConnectedToThreeBrokers();

      getBroker(BROKER_A_NAME).stop();
      getBroker(BROKER_A_NAME).waitUntilStopped();
      removeBroker(BROKER_A_NAME);

      Thread.sleep(5000);

      assertClientsConnectedToTwoBrokers();

      createBrokerA(false, null, null, null);
      getBroker(BROKER_A_NAME).waitUntilStarted();
      Thread.sleep(5000);

      assertClientsConnectedToThreeBrokers();
   }

   /**
    * Test to ensure clients are evenly to all available brokers in the
    * network.
    *
    * @throws Exception
    */
   public void testThreeBrokerClusterClientDistributions() throws Exception {

      initSingleTcBroker("", null, null);

      Thread.sleep(2000);
      setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false&initialReconnectDelay=500");
      createClients(100);
      Thread.sleep(5000);

      runClientDistributionTests(false, null, null, null);
   }

   /**
    * Test to verify that clients are distributed with no less than 20% of the
    * clients on any one broker.
    *
    * @throws Exception
    */
   public void testThreeBrokerClusterDestinationFilter() throws Exception {

      initSingleTcBroker("", null, null);

      Thread.sleep(2000);
      setClientUrl("failover://(" + BROKER_A_CLIENT_TC_ADDRESS + "," + BROKER_B_CLIENT_TC_ADDRESS + ")");
      createClients();

      runTests(false, null, null, "Queue.TEST.FOO.>");
   }

   public void testFailOverWithUpdateClientsOnRemove() throws Exception {
      // Broker A
      addBroker(BROKER_A_NAME, createBroker(BROKER_A_NAME));
      TransportConnector connectorA = getBroker(BROKER_A_NAME).addConnector(BROKER_A_CLIENT_TC_ADDRESS);
      connectorA.setName("openwire");
      connectorA.setRebalanceClusterClients(true);
      connectorA.setUpdateClusterClients(true);
      connectorA.setUpdateClusterClientsOnRemove(true); //If set to false the test succeeds.
      addNetworkBridge(getBroker(BROKER_A_NAME), "A_2_B_Bridge", "static://(" + BROKER_B_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
      getBroker(BROKER_A_NAME).start();

      // Broker B
      addBroker(BROKER_B_NAME, createBroker(BROKER_B_NAME));
      TransportConnector connectorB = getBroker(BROKER_B_NAME).addConnector(BROKER_B_CLIENT_TC_ADDRESS);
      connectorB.setName("openwire");
      connectorB.setRebalanceClusterClients(true);
      connectorB.setUpdateClusterClients(true);
      connectorB.setUpdateClusterClientsOnRemove(true); //If set to false the test succeeds.
      addNetworkBridge(getBroker(BROKER_B_NAME), "B_2_A_Bridge", "static://(" + BROKER_A_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
      getBroker(BROKER_B_NAME).start();

      getBroker(BROKER_B_NAME).waitUntilStarted();
      Thread.sleep(1000);

      // create client connecting only to A. It should receive broker B address whet it connects to A.
      setClientUrl("failover:(" + BROKER_A_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=true");
      createClients(1);
      Thread.sleep(5000);

      // We stop broker A.
      logger.info("Stopping broker A whose address is: {}", BROKER_A_CLIENT_TC_ADDRESS);
      getBroker(BROKER_A_NAME).stop();
      getBroker(BROKER_A_NAME).waitUntilStopped();
      Thread.sleep(5000);

      // Client should failover to B.
      assertAllConnectedTo(BROKER_B_CLIENT_TC_ADDRESS);
   }

   /**
    * Runs a 3 Broker dynamic failover test: <br/>
    * <ul>
    * <li>asserts clients are distributed across all 3 brokers</li>
    * <li>asserts clients are distributed across 2 brokers after removing the 3rd</li>
    * <li>asserts clients are distributed across all 3 brokers after
    * reintroducing the 3rd broker</li>
    * </ul>
    *
    * @param multi
    * @param tcParams
    * @param clusterFilter
    * @param destinationFilter
    * @throws Exception
    * @throws InterruptedException
    */
   private void runTests(boolean multi,
                         String tcParams,
                         String clusterFilter,
                         String destinationFilter) throws Exception, InterruptedException {
      assertClientsConnectedToThreeBrokers();

      getBroker(BROKER_C_NAME).stop();
      getBroker(BROKER_C_NAME).waitUntilStopped();
      removeBroker(BROKER_C_NAME);

      Thread.sleep(5000);

      assertClientsConnectedToTwoBrokers();

      createBrokerC(multi, tcParams, clusterFilter, destinationFilter);
      getBroker(BROKER_C_NAME).waitUntilStarted();
      Thread.sleep(5000);

      assertClientsConnectedToThreeBrokers();
   }

   /**
    * @param multi
    * @param tcParams
    * @param clusterFilter
    * @param destinationFilter
    * @throws Exception
    * @throws InterruptedException
    */
   private void runClientDistributionTests(boolean multi,
                                           String tcParams,
                                           String clusterFilter,
                                           String destinationFilter) throws Exception, InterruptedException {
      assertClientsConnectedToThreeBrokers();
      assertClientsConnectionsEvenlyDistributed(.25);

      getBroker(BROKER_C_NAME).stop();
      getBroker(BROKER_C_NAME).waitUntilStopped();
      removeBroker(BROKER_C_NAME);

      Thread.sleep(5000);

      assertClientsConnectedToTwoBrokers();
      assertClientsConnectionsEvenlyDistributed(.35);

      createBrokerC(multi, tcParams, clusterFilter, destinationFilter);
      getBroker(BROKER_C_NAME).waitUntilStarted();
      Thread.sleep(5000);

      assertClientsConnectedToThreeBrokers();
      assertClientsConnectionsEvenlyDistributed(.20);
   }

   @Override
   protected void setUp() throws Exception {
   }

   @Override
   protected void tearDown() throws Exception {
      shutdownClients();
      Thread.sleep(2000);
      destroyBrokerCluster();
   }

   private void initSingleTcBroker(String params, String clusterFilter, String destinationFilter) throws Exception {
      createBrokerA(false, params, clusterFilter, null);
      createBrokerB(false, params, clusterFilter, null);
      createBrokerC(false, params, clusterFilter, null);
      getBroker(BROKER_C_NAME).waitUntilStarted();
   }

   private void initMultiTcCluster(String params, String clusterFilter) throws Exception {
      createBrokerA(true, params, clusterFilter, null);
      createBrokerB(true, params, clusterFilter, null);
      createBrokerC(true, params, clusterFilter, null);
      getBroker(BROKER_C_NAME).waitUntilStarted();
   }

   private void createBrokerA(boolean multi,
                              String params,
                              String clusterFilter,
                              String destinationFilter) throws Exception {
      final String tcParams = (params == null) ? "" : params;
      if (getBroker(BROKER_A_NAME) == null) {
         addBroker(BROKER_A_NAME, createBroker(BROKER_A_NAME));
         addTransportConnector(getBroker(BROKER_A_NAME), "openwire", BROKER_A_CLIENT_TC_ADDRESS + tcParams, true);
         if (multi) {
            addTransportConnector(getBroker(BROKER_A_NAME), "network", BROKER_A_NOB_TC_ADDRESS + tcParams, false);
            addNetworkBridge(getBroker(BROKER_A_NAME), "A_2_B_Bridge", "static://(" + BROKER_B_NOB_TC_ADDRESS + ")?useExponentialBackOff=false", false, clusterFilter);
            addNetworkBridge(getBroker(BROKER_A_NAME), "A_2_C_Bridge", "static://(" + BROKER_C_NOB_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
         }
         else {
            addNetworkBridge(getBroker(BROKER_A_NAME), "A_2_B_Bridge", "static://(" + BROKER_B_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, clusterFilter);
            addNetworkBridge(getBroker(BROKER_A_NAME), "A_2_C_Bridge", "static://(" + BROKER_C_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
         }
         getBroker(BROKER_A_NAME).start();
      }
   }

   private void createBrokerB(boolean multi,
                              String params,
                              String clusterFilter,
                              String destinationFilter) throws Exception {
      final String tcParams = (params == null) ? "" : params;
      if (getBroker(BROKER_B_NAME) == null) {
         addBroker(BROKER_B_NAME, createBroker(BROKER_B_NAME));
         addTransportConnector(getBroker(BROKER_B_NAME), "openwire", BROKER_B_CLIENT_TC_ADDRESS + tcParams, true);
         if (multi) {
            addTransportConnector(getBroker(BROKER_B_NAME), "network", BROKER_B_NOB_TC_ADDRESS + tcParams, false);
            addNetworkBridge(getBroker(BROKER_B_NAME), "B_2_A_Bridge", "static://(" + BROKER_A_NOB_TC_ADDRESS + ")?useExponentialBackOff=false", false, clusterFilter);
            addNetworkBridge(getBroker(BROKER_B_NAME), "B_2_C_Bridge", "static://(" + BROKER_C_NOB_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
         }
         else {
            addNetworkBridge(getBroker(BROKER_B_NAME), "B_2_A_Bridge", "static://(" + BROKER_A_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, clusterFilter);
            addNetworkBridge(getBroker(BROKER_B_NAME), "B_2_C_Bridge", "static://(" + BROKER_C_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
         }
         getBroker(BROKER_B_NAME).start();
      }
   }

   private void createBrokerC(boolean multi,
                              String params,
                              String clusterFilter,
                              String destinationFilter) throws Exception {
      final String tcParams = (params == null) ? "" : params;
      if (getBroker(BROKER_C_NAME) == null) {
         addBroker(BROKER_C_NAME, createBroker(BROKER_C_NAME));
         addTransportConnector(getBroker(BROKER_C_NAME), "openwire", BROKER_C_CLIENT_TC_ADDRESS + tcParams, true);
         if (multi) {
            addTransportConnector(getBroker(BROKER_C_NAME), "network", BROKER_C_NOB_TC_ADDRESS + tcParams, false);
            addNetworkBridge(getBroker(BROKER_C_NAME), "C_2_A_Bridge", "static://(" + BROKER_A_NOB_TC_ADDRESS + ")?useExponentialBackOff=false", false, clusterFilter);
            addNetworkBridge(getBroker(BROKER_C_NAME), "C_2_B_Bridge", "static://(" + BROKER_B_NOB_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
         }
         else {
            addNetworkBridge(getBroker(BROKER_C_NAME), "C_2_A_Bridge", "static://(" + BROKER_A_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, clusterFilter);
            addNetworkBridge(getBroker(BROKER_C_NAME), "C_2_B_Bridge", "static://(" + BROKER_B_CLIENT_TC_ADDRESS + ")?useExponentialBackOff=false", false, null);
         }
         getBroker(BROKER_C_NAME).start();
      }
   }
}
