//$Id: TOTAL_TOKEN.java,v 1.14 2006/04/28 15:25:00 belaban Exp $

package org.jgroups.protocols;

import org.jgroups.*;
import org.jgroups.blocks.GroupRequest;
import org.jgroups.protocols.pbcast.Digest;
import org.jgroups.protocols.ring.RingNodeFlowControl;
import org.jgroups.protocols.ring.RingToken;
import org.jgroups.protocols.ring.TokenLostException;
import org.jgroups.protocols.ring.UdpRingNode;
import org.jgroups.stack.IpAddress;
import org.jgroups.stack.RpcProtocol;
import org.jgroups.util.RspList;
import org.jgroups.util.Util;

import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.*;

/**
 * <p>
 * Total order implementation based on <a href="http://citeseer.nj.nec.com/amir95totem.html">
 * The Totem Single-Ring Ordering and Membership Protocol</a>.
 * <p>
 *
 * <p>
 * However, this is an adaption of algorithm mentioned in the research paper above since we reuse
 * our own membership protocol and failure detectors. Somewhat different flow control mechanism is
 * also implemented.
 *
 * <p>
 * Token passing is done through reliable point-to-point udp channels provided by UNICAST layer.
 * Process groups nodes members are organized in a logical ring.
 * </p>
 *
 * <p>
 * Total token layer doesn't need NAKACK nor STABLE layer beneath it since it implements it's own
 * retransmission and tracks stability of the messages from the information piggybacked on the
 * token itself.
 * </p>
 *
 * <p>
 * For the typical protocol stack configuration used, see org.jgroups.demos.TotalTokenDemo and
 * total-token.xml configuration file provided with this distribution of JGroups.
 * </p>
 *
 *
 *
 *@author Vladimir Blagojevic vladimir@cs.yorku.ca
 *@version $Revision: 1.14 $
 *
 *@see org.jgroups.protocols.ring.RingNodeFlowControl
 *@see org.jgroups.protocols.ring.RingNode
 *@see org.jgroups.protocols.ring.TcpRingNode
 *@see org.jgroups.protocols.ring.UdpRingNode
 *
 **/


public class TOTAL_TOKEN extends RpcProtocol
{
    private static final Object[] NULL_OBJ=new Object[]{};
    private static final Class[] NULL_TYPES=new Class[]{};


    public static class TotalTokenHeader extends Header
    {


       /**
        * sequence number of the message
        */
       private long seq;


       /**
        *used for externalization
        */
       public TotalTokenHeader()
       {
       }

       public TotalTokenHeader(long seq)
       {
          this.seq = seq;
       }

       public TotalTokenHeader(Long seq)
       {
          this.seq = seq.longValue();
       }


       /**
        *Returns sequence number of the message that owns this header
        *@return sequence number
        */
       public long getSeq()
       {
          return seq;
       }

       /**
        *Returns size of the header
        * @return headersize in bytes
        */
       public long size()
       {
          //calculated using Util.SizeOf(Object)
          return 121;
       }

       /**
        * Manual serialization
        *
        *
        */
       public void writeExternal(ObjectOutput out) throws IOException
       {
          out.writeLong(seq);
       }

       /**
        * Manual deserialization
        *
        */
       public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException
       {
          seq = in.readLong();
       }

       public String toString()
       {
          return "[TotalTokenHeader=" + seq + ']';
       }
    }

   public static class RingTokenHeader extends Header
   {
      public RingTokenHeader()
      {
      }

      public void writeExternal(ObjectOutput out) throws IOException
      {
      }

      public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException
      {
      }

      public long size()
      {
         //calculated using Util.SizeOf(Object)
         return 110;
      }
   }


   private static final int OPERATIONAL_STATE = 0;
   private static final int RECOVERY_STATE = 1;

   UdpRingNode node;
   RingNodeFlowControl flowControl;
   Address localAddress;
   private final TokenTransmitter tokenRetransmitter=new TokenTransmitter();
   final List newMessagesQueue = Collections.synchronizedList(new ArrayList());
   SortedSet liveMembersInRecovery,suspects;

   final Object mutex = new Object();
   TreeMap receivedMessagesQueue;
   long myAru = 0;

   final Object threadCoordinationMutex = new Object();
   final boolean tokenInStack = false;
   final boolean threadDeliveringMessage = false;
   boolean tokenSeen = false;


   volatile boolean isRecoveryLeader = false;
   volatile int state;
   volatile int sleepTime = 10;

   long highestSeenSeq = 0;
   long lastRoundTokensAru = 0;
   int lastRoundTransmitCount,lastRoundRebroadcastCount = 0;
   int blockSendingBacklogThreshold = Integer.MAX_VALUE;
   int unblockSendingBacklogThreshold = Integer.MIN_VALUE;
   boolean tokenCirculating = false;
   boolean senderBlocked = false;
    final Object block_sending=new Object();
   public static final String prot_name = "TOTAL_TOKEN";


   public String getName()
   {
      return prot_name;
   }

   private String getState()
   {
      if (state == OPERATIONAL_STATE)
      {
         return "OPERATIONAL";
      }
      else
         return "RECOVERY";
   }


    public void start() throws Exception {
        super.start();
        receivedMessagesQueue = new TreeMap();
        tokenRetransmitter.start();
    }

   /**
    * Overrides @org.jgroups.stack.MessageProtocol#stop().
    */
   public void stop()
   {
       super.stop();
       tokenRetransmitter.shutDown();
   }




   /**
    * Setup the Protocol instance acording to the configuration string
    *
    */
   public boolean setProperties(Properties props)
   {
      String str;

       super.setProperties(props);
      str = props.getProperty("block_sending");
      if (str != null)
      {
         blockSendingBacklogThreshold = Integer.parseInt(str);
         props.remove("block_sending");
      }

      str = props.getProperty("unblock_sending");
      if (str != null)
      {
         unblockSendingBacklogThreshold = Integer.parseInt(str);
         props.remove("unblock_sending");
      }

      if (props.size() > 0)
      {
         log.error("UDP.setProperties(): the following properties are not recognized: " + props);

         return false;
      }
      return true;
   }

   public IpAddress getTokenReceiverAddress()
   {
      return node != null? node.getTokenReceiverAddress() : null;
   }

   public Vector providedUpServices()
   {
      Vector retval = new Vector();
      retval.addElement(new Integer(Event.GET_DIGEST));
      retval.addElement(new Integer(Event.GET_DIGEST_STATE));
      retval.addElement(new Integer(Event.SET_DIGEST));
      return retval;
   }

   public boolean handleUpEvent(Event evt)
   {
      Message msg;
      Header h;
      switch (evt.getType())
      {

         case Event.SET_LOCAL_ADDRESS:
            localAddress = (Address) evt.getArg();
            node = new UdpRingNode(this, localAddress);
            flowControl = new RingNodeFlowControl();
            break;

         case Event.SUSPECT:
            Address suspect = (Address) evt.getArg();
            onSuspectMessage(suspect);
            break;

         case Event.MSG:
            msg = (Message) evt.getArg();
            h = msg.getHeader(getName());
            if (h instanceof TotalTokenHeader)
            {
               messageArrived(msg);
               return false;
            }
            else if (h instanceof RingTokenHeader)
            {
                if(node != null) {
                    Object tmp=msg.getObject();
                    node.tokenArrived(tmp);
                }
               return false;
            }
      }
      return true;
   }

   public boolean handleDownEvent(Event evt)
   {
      switch (evt.getType())
      {
         case Event.GET_DIGEST:
         case Event.GET_DIGEST_STATE:

            Digest d = new Digest(members.size());
            Address sender = null;
            //all members have same digest :)
            for (int j = 0; j < members.size(); j++)
            {
               sender = (Address) members.elementAt(j);
               d.add(sender, highestSeenSeq, highestSeenSeq);
            }
            passUp(new Event(Event.GET_DIGEST_OK, d));
            return false;
         case Event.SET_DIGEST:
            Digest receivedDigest = (Digest) evt.getArg();

          // changed by bela July 12 2005, not sure if this is correct, don't know what the original author
          // intended to do here
            // myAru = receivedDigest.highSeqnoAt(0);
            myAru = receivedDigest.highSeqnoAt(localAddress);
            return false;

         case Event.VIEW_CHANGE:
            onViewChange();
            return true;

            /*
         case Event.CLEANUP:
            // do not pass cleanup event
            //further down. This is a hack to enable
            // sucessfull leave from group when using pbcast.GMS.
            // It just buys us 5 seconds to imminent STOP
            // event following CLEANUP. We hope that the moment
            // this node disconnect up until new view is installed
            // at other members is less than 5 seconds.

            //The proper way would be to:
            //trap DISCONNECT event on the way down, do not pass it further.
            //wait for the new view to be installed (effectively excluding this node out of
            //ring) , wait for one token roundtrip time, and then send that trapped
            //DISCONNECT event down furhter to generate DISCONNECT_OK on the way up.
            // CLEANUP and STOP are generated after DISCONNECT.

            //However, as the things stand right now pbcast.GMS stops working immediately
            //when it receives DISCONNECT thus the new view is never generated in node that is
            //leaving the group.

            //pbcsat.GMS should still generate new view and stop working when
            //it receives STOP event.

            //In timeline DISCONNECT < CLEANUP < STOP
            return false;
            */

         case Event.MSG:
            Message msg = (Message) evt.getArg();
            //handle only multicasts
            if (msg == null) return false;
            if (msg.getDest() == null || msg.getDest().isMulticastAddress())
            {
               newMessagesQueue.add(msg);
               return false;
            }
      }
      return true;
   }

   private void onViewChange()
   {
      isRecoveryLeader = false;

      if (suspects != null)
      {
         suspects.clear();
         suspects = null;
      }
      if (liveMembersInRecovery != null)
      {
         liveMembersInRecovery.clear();
         liveMembersInRecovery = null;
      }
   }

   private void onSuspectMessage(Address suspect)
   {
      state = RECOVERY_STATE;
      if (suspects == null || suspects.size() == 0)
      {
         suspects = Collections.synchronizedSortedSet(new TreeSet());
         liveMembersInRecovery = Collections.synchronizedSortedSet(new TreeSet(members));
      }
      suspects.add(suspect);
      liveMembersInRecovery.removeAll(suspects);
      isRecoveryLeader = isRecoveryLeader(liveMembersInRecovery);
   }

   /**
    * Given a set of surviving members in the transitioanl view
    * returns true if this stack is elected to be recovery leader.
    *
    */
   private boolean isRecoveryLeader(SortedSet liveMembers)
   {
      boolean recoveryLeader = false;
      if (liveMembers.size() > 0)
      {
         recoveryLeader = localAddress.equals(liveMembers.first());
      }

       if(log.isInfoEnabled()) log.info("live memebers are " + liveMembers);
       if(log.isInfoEnabled()) log.info("I am recovery leader?" + recoveryLeader);
       return recoveryLeader;

   }

   public long getAllReceivedUpTo()
   {
      return myAru;
   }

   public void installTransitionalView(Vector members)
   {
       if(node != null)
           node.reconfigure(members);
   }

   /**
    *  Total Token recovery protocol (TTRP)
    *
    *
    *
    *  Upon transition to recovery state, coordinator sends multiple reliable
    *  unicasts message requesting each ring member to report it's allReceivedUpto
    *  value. When all replies are received, a response list of allReceivedUpto
    *  values is sorted and transformed into a set while dropping the lowest
    *  allReceivedUpto value. For example , received response list  [4,4,5,6,7,7,8]
    *  is transformed into [5,6,7,8] thus not including the lowest value 4.
    *
    *  The objective of the recovery protocol is to have each member receive all
    *  messages up to maximum sequence value M from the response list, thus
    *  satisfying virtual synchrony properties.
    *
    *  Note that if all members report the same allReceivedUpto values, then
    *  virtual synchrony is satisfied (since all surviving members have seen
    *  the same set of messages) and we can immediately inject operational
    *  token which will enable installment of the new view.
    *
    *  Otherwise, a constructed set S of all allReceivedUpto values represent sequence ids
    *  of messages that have to be received by all mebers prior to installing new
    *  view thus satisfying virtual synchrony properties.
    *
    *  A transitional view, visible only to TOTAL_TOKEN layer is then installed
    *  on the ring by a coordinator. Again a multiple unicast are used. A
    *  transitional view is deduced from current view by excluding suspected members.
    *  Coordinator creates a recovery token by appending the set S of sequence ids to
    *  token retransmission request list. Recovery token is then inserted into
    *  transitional ring view.
    *
    *  Upon reception of recovery token, ring members are not allowed to transmit
    *  any additional new messages but only to retransmit messages from the
    *  specified token retransmission request list.
    *
    *  When all member detect that they have received all messages upto sequence
    *  value M , the next member that first receives token converts it to operatioanl
    *  token and  normal operational state is restored in all nodes.
    *
    *  If token is lost during recovery stage, recovery protocol is restarted.
    *
    * */
   private void recover()
   {

      if (isRecoveryLeader && state == RECOVERY_STATE)
      {

          if(log.isInfoEnabled()) log.info("I am starting recovery now");

          Vector m = new Vector(liveMembersInRecovery);

          RspList list=callRemoteMethods(m, "getAllReceivedUpTo", NULL_OBJ, NULL_TYPES, GroupRequest.GET_ALL, 0);
         //RspList list = callRemoteMethods(m, "getAllReceivedUpTo", GroupRequest.GET_ALL, 0);
         Vector myAllReceivedUpTos = list.getResults();

          callRemoteMethods(m, "getAllReceivedUpTo", NULL_OBJ, NULL_TYPES, GroupRequest.GET_ALL, 0);
         //callRemoteMethods(m, "getAllReceivedUpTo", GroupRequest.GET_ALL, 0);
         Vector myAllReceivedUpTosConfirm = list.getResults();


         while (!myAllReceivedUpTos.equals(myAllReceivedUpTosConfirm))
         {
            myAllReceivedUpTos = myAllReceivedUpTosConfirm;
             callRemoteMethods(m, "getAllReceivedUpTo", NULL_OBJ, NULL_TYPES, GroupRequest.GET_ALL, 0);
            // callRemoteMethods(m, "getAllReceivedUpTo", GroupRequest.GET_ALL, 0);
            myAllReceivedUpTosConfirm = list.getResults();

             if(log.isInfoEnabled()) log.info("myAllReceivedUpto values are" + myAllReceivedUpTos);
             if(log.isInfoEnabled()) log.info("myAllReceivedUpto confirm values are " + myAllReceivedUpTosConfirm);
         }


          if(log.isInfoEnabled()) log.info("myAllReceivedUpto stabilized values are" + myAllReceivedUpTos);
          if(log.isInfoEnabled()) log.info("installing transitional view to repair the ring...");

          callRemoteMethods(m, "installTransitionalView", new Object[]{m}, new String[]{Vector.class.getName()},
                            GroupRequest.GET_ALL, 0);
         //callRemoteMethods(m, "installTransitionalView", m, GroupRequest.GET_ALL, 0);

         Vector xmits = prepareRecoveryRetransmissionList(myAllReceivedUpTos);
         RingToken injectToken = null;
         if (xmits.size() > 1)
         {

             if(log.isInfoEnabled()) log.info("VS not satisfied, injecting recovery token...");
             long aru = ((Long) xmits.firstElement()).longValue();
            long highest = ((Long) xmits.lastElement()).longValue();

            injectToken = new RingToken(RingToken.RECOVERY);
            injectToken.setHighestSequence(highest);
            injectToken.setAllReceivedUpto(aru);


            Collection rtr = injectToken.getRetransmissionRequests();
            rtr.addAll(xmits);
         }
         else
         {

             if(log.isInfoEnabled()) log.info("VS satisfied, injecting operational token...");
             injectToken = new RingToken();
            long sequence = ((Long) xmits.firstElement()).longValue();
            injectToken.setHighestSequence(sequence);
            injectToken.setAllReceivedUpto(sequence);
         }
          if(node != null)
              node.passToken(injectToken);
         tokenRetransmitter.resetTimeout();
      }
   }

   /**
    * Prepares a retransmissions list for recovery protocol
    * given a collection of all myReceivedUpTo values as
    * reported by polled surviving members.
    *
    *
    *
    */
   private Vector prepareRecoveryRetransmissionList(Vector sequences)
   {
      Collections.sort(sequences);
      Long first = (Long) sequences.firstElement();
      Long last = (Long) sequences.lastElement();


      Vector retransmissions = new Vector();
      if (first.equals(last))
      {
         retransmissions.add(new Long(first.longValue()));
      }
      else
      {
         for (long j = first.longValue() + 1; j <= last.longValue(); j++)
         {
            retransmissions.add(new Long(j));
         }
      }
      return retransmissions;
   }


   protected void updateView(View newMembers)
   {
      super.updateView(newMembers);
      Vector newViewMembers = newMembers.getMembers();
      flowControl.viewChanged(newViewMembers.size());
       if(node != null)
           node.reconfigure(newViewMembers);
      boolean isCoordinator = localAddress.equals(newViewMembers.firstElement());
      int memberSize = newViewMembers.size();

      if (memberSize == 1 && isCoordinator && !tokenCirculating)
      {
         //create token for the first time , lets roll
         tokenCirculating = true;
         RingToken token = new RingToken();
          if(node != null)
              node.passToken(token);
         tokenRetransmitter.resetTimeout();
      }
      sleepTime = (20/memberSize);
   }


   /**
    * TOTAL_TOKEN's up-handler thread invokes this method after multicast
    * message originating from some other TOTAL_TOKEN stack layer arrives at
    * this stack layer.
    *
    * Up-handler thread coordinates it's access to a shared variables
    * with TokenTransmitter thread.
    *
    * See tokenReceived() for details.
    *
    */
   private void messageArrived(Message m)
   {
      TotalTokenHeader h = (TotalTokenHeader) m.getHeader(getName());
      long seq = h.getSeq();


      synchronized (mutex)
      {
         if ((myAru + 1) <= seq)
         {
            if (seq > highestSeenSeq)
            {
               highestSeenSeq = seq;
            }

            receivedMessagesQueue.put(new Long(seq), m);
            if ((myAru + 1) == seq)
            {
               myAru = seq;
               passUp(new Event(Event.MSG, m));
            }
            if (isReceiveQueueHolePlugged())
            {
               myAru = deliverMissingMessages();
            }
         }
      }
   }

   /**
    * Returns true if there is a hole in receive queue and at
    * least one messages with sequence id consecutive to myAru.
    *
    *
    */
   private boolean isReceiveQueueHolePlugged()
   {
      return ((myAru < highestSeenSeq) && receivedMessagesQueue.containsKey(new Long(myAru + 1)));
   }


   /**
    * Delivers as much as possible messages from receive
    * message queue as long as they are consecutive with
    * respect to their sequence ids.
    *
    */
   private long deliverMissingMessages()
   {
      Map.Entry entry = null;
      boolean inOrder = true;
      long lastDelivered = myAru;
      Set deliverySet = receivedMessagesQueue.tailMap(new Long(myAru + 1)).entrySet();


       if(log.isInfoEnabled()) log.info("hole getting plugged, prior muAru " + myAru);


       for (Iterator iterator = deliverySet.iterator();inOrder && iterator.hasNext();)
       {
          entry = (Map.Entry) iterator.next();
          long nextInQueue = ((Long) entry.getKey()).longValue();
          if (lastDelivered + 1 == nextInQueue)
          {
             Message m = (Message) entry.getValue();
             passUp(new Event(Event.MSG, m));
             lastDelivered++;
          }
          else
          {
             inOrder = false;
          }
       }


       if(log.isInfoEnabled()) log.info("hole getting plugged, post muAru " + lastDelivered);
       return lastDelivered;
   }

   /**
    * Checks if the receivedMessageQueue has any missing sequence
    * numbers in it, and if it does it finds holes in sequences from
    * this stack's receivedMessageQueue and adds them to token retransmission
    * list, thus informing other group members about messages missing
    * from this stack.
    *
    *
    */
   private void updateTokenRtR(RingToken token)
   {
      long holeLowerBound = 0;
      long holeUpperBound = 0;
      Long missingSequence = null;
      Collection retransmissionList = null;


      //any holes?
      if (myAru < token.getHighestSequence())
      {
         retransmissionList = token.getRetransmissionRequests();
         Set received = receivedMessagesQueue.tailMap(new Long(myAru + 1)).keySet();
         Iterator nonMissing = received.iterator();
         holeLowerBound = myAru;


            if(log.isDebugEnabled()) log.debug("retransmission request prior" + retransmissionList);

         while (nonMissing.hasNext())
         {
            Long seq = (Long) nonMissing.next();
            holeUpperBound = seq.longValue();

            while (holeLowerBound < holeUpperBound)
            {
               missingSequence = new Long(++holeLowerBound);
               retransmissionList.add(missingSequence);
            }
            holeLowerBound = holeUpperBound;
         }

         holeUpperBound = token.getHighestSequence();
         while (holeLowerBound < holeUpperBound)
         {
            missingSequence = new Long(++holeLowerBound);
            retransmissionList.add(missingSequence);
         }


            if(log.isDebugEnabled()) log.debug("retransmission request after" + retransmissionList);
      }
   }


   /**
    * Sends messages in this stacks's outgoing queue and
    * saves a copy of each outgoing message in case they got lost.
    * If messages get lost it is thus guaranteed that each stack
    * that sent any message has a copy of it ready for retransmitting.
    *
    * Each sent message is stamped by monotonically increasing
    * sequence number starting from the highest sequence "seen"
    * on the ring.
    *
    * Returns number of messages actually sent.  The number of
    * sent messages is bounded above by the flow control
    * algorithm (allowedCount) and bounded below by the number
    * of pending messages in newMessagesQueue.
    */
   private int broadcastMessages(int allowedCount, RingToken token)
   {
      List sendList = null;
      synchronized (newMessagesQueue)
      {
         int queueSize = newMessagesQueue.size();

         if (queueSize <= 0)
         {
            return 0;
         }

         else if (queueSize > allowedCount)
         {
            sendList = new ArrayList(newMessagesQueue.subList(0, allowedCount));
            newMessagesQueue.removeAll(sendList);
         }

         else
         {
            sendList = new ArrayList();
            sendList.addAll(newMessagesQueue);
            newMessagesQueue.clear();
         }
      }

      long tokenSeq = token.getHighestSequence();

      for (Iterator iterator = sendList.iterator(); iterator.hasNext();)
      {
         Message m = (Message) iterator.next();
         m.setSrc(localAddress);
         m.setDest(null); // mcast address
         m.putHeader(getName(), new TotalTokenHeader(++tokenSeq));
         receivedMessagesQueue.put(new Long(tokenSeq), m);
         passDown(new Event(Event.MSG, m));
      }

      if (token.getHighestSequence() == token.getAllReceivedUpto())
      {
         token.setAllReceivedUpto(tokenSeq);
      }
      token.setHighestSequence(tokenSeq);
      return sendList.size();
   }


   /**
    * TokenTransmitter thread runs this method after receiving token.
    * Thread is possibly blocked if up-handler thread is currently running
    * through this stack i.e delivering an Event. Up-hanlder thread will
    * notify blocked TokenTransmitter thread when it has delivered current
    * Event so TokenTransmitter can proceed.
    * TokenTransmitter thread in turn notifies possibly blocked up-handler thread
    * when token has left the stack. Thus TokenTransmitter and up-hadler thread
    * coordinate their access to shared variables(receivedMessageQueue and myAru).
    *
    * tokenReceived method and subsequent methods called from tokenReceived represent
    * in some parts the totaly ordered algorithm presented in Amir's paper (see class
    * header for link)
    *
    *
    *
    */
   private void tokenReceived(RingToken token)
   {


       if(log.isInfoEnabled()) log.info(token.toString());
       if(log.isDebugEnabled()) log.debug(getState());


       flowControl.setBacklog(newMessagesQueue.size());
      flowControl.updateWindow(token);


      blockSenderIfRequired();
      unBlockSenderIfAcceptable();


      long tokensAru = 0;
      int broadcastCount = 0;
      int rebroadcastCount = 0;
      synchronized (mutex)
      {
         if (!tokenSeen)
         {
            long lastRoundAru = token.getHighestSequence() - token.getLastRoundBroadcastCount();
            if (myAru < token.getAllReceivedUpto())
            {
               myAru = lastRoundAru;
            }
            //if(log.isInfoEnabled()) log.info("TOTAL_TOKEN.tokenReceived()", "tokenSeen " + myAru);
            tokenSeen = true;
         }

         if (token.getType() == RingToken.RECOVERY)
         {
            highestSeenSeq = token.getHighestSequence();
            if (highestSeenSeq == myAru)
            {
               if(log.isInfoEnabled()) log.info("member node recovered");
               token.addRecoveredMember(localAddress);
            }
         }

         updateTokenRtR(token);

         int allowedToBroadcast = flowControl.getAllowedToBroadcast(token);
         rebroadcastCount = rebroadcastMessages(token);
         allowedToBroadcast -= rebroadcastCount;


          if(log.isInfoEnabled()) log.info("myAllReceivedUpto" + myAru);
          if(log.isInfoEnabled()) log.info("allowedToBroadcast" + allowedToBroadcast);
          if(log.isInfoEnabled()) log.info("newMessagesQueue.size()" + newMessagesQueue.size());

          tokensAru = token.getAllReceivedUpto();

         if (myAru < tokensAru ||
                 localAddress.equals(token.getAruId()) ||
                 token.getAruId() == null)
         {
            token.setAllReceivedUpto(myAru);
            if (token.getAllReceivedUpto() == token.getHighestSequence())
            {
               token.setAruId(null);
            }
            else
            {
               token.setAruId(localAddress);
            }
         }
         if (allowedToBroadcast > 0 && token.getType() == RingToken.OPERATIONAL)
         {
            broadcastCount = broadcastMessages(allowedToBroadcast, token);
         }

         if (tokensAru > lastRoundTokensAru)
         {
            removeStableMessages(receivedMessagesQueue, lastRoundTokensAru);
         }

      } //end synchronized

      //give CPU some breath
      Util.sleep(sleepTime);

      token.incrementTokenSequence();
      token.addLastRoundBroadcastCount(broadcastCount - lastRoundTransmitCount);
      token.addBacklog(flowControl.getBacklogDifference());
      flowControl.setPreviousBacklog();
      lastRoundTransmitCount = broadcastCount;
      lastRoundRebroadcastCount = rebroadcastCount;
      lastRoundTokensAru = tokensAru;
   }

   /**
    *
    * Rebroadcasts messages specified in token's retransmission
    * request list if those messages are available in this stack.
    * Returns number of rebroadcasted messages.
    *
    */
   private int rebroadcastMessages(RingToken token)
   {
      int rebroadCastCount = 0;
      Collection rexmitRequests = token.getRetransmissionRequests();
      if (rexmitRequests.size() > 0)
      {
         Collection rbl = getRebroadcastList(rexmitRequests);
         rebroadCastCount = rbl.size();
         if (rebroadCastCount > 0)
         {

             if(log.isInfoEnabled()) log.info("rebroadcasting " + rbl);

             Long s = null;
            for (Iterator iterator = rbl.iterator(); iterator.hasNext();)
            {
               s = (Long) iterator.next();
               Message m = (Message) receivedMessagesQueue.get(s);
               passDown(new Event(Event.MSG, m));
            }
         }
      }
      return rebroadCastCount;
   }


   private void invalidateOnTokenloss()
   {
      lastRoundTransmitCount = 0;
      flowControl.invalidate();
   }

   /**
    * Checks if the down pending queue's (newMessagesQueue) size is
    * greater than blockSendingBacklogThreshold specified in the properties.
    * If it is, client's sending thread is effectively blocked until
    * down pending queue's size drops below unblockSendingBacklogThreshold.
    *
    *
    */
   private void blockSenderIfRequired()
   {
      if (!senderBlocked && flowControl.getBacklog() > blockSendingBacklogThreshold)
      {
          synchronized(block_sending) {
              senderBlocked = true;
              while(senderBlocked) {
                  try {
                      block_sending.wait();
                  }
                  catch(InterruptedException e) {
                  }
              }
          }
      }
   }

   /**
    * Checks if the down pending queue's (newMessagesQueue) size is
    * smaller than unblockSendingBacklogThreshold specified in the properties.
    * If it is, client's sending thread is effectively unblocked enabling
    * new messages to be queued for transmission.
    *
    *
    */
   private void unBlockSenderIfAcceptable()
   {
      if (senderBlocked && flowControl.getBacklog() < unblockSendingBacklogThreshold)
      {
          synchronized(block_sending) {
              senderBlocked = false;
              block_sending.notifyAll();
          }
      }

   }

   /**
    * Removes messages determined to be stable(i.e seen by all members)
    * from the specified queue.  If the client also clears all reference
    * to these messages (in application space) they become eligible for garabge
    * collection.
    *
    *
    */

   private void removeStableMessages(TreeMap m, long upToSeq)
   {

      if (m.size() > 0)
      {
         long first = ((Long) m.firstKey()).longValue();
         if (first > upToSeq)
         {
            upToSeq = first;
         }


          if(log.isDebugEnabled()) log.debug("cutting queue first key " + m.firstKey()
                      + " cut at " + upToSeq + " last key " + m.lastKey());
          SortedMap stable = m.headMap(new Long(upToSeq));
         stable.clear();
      }
   }

   /**
    * Determines a subset of message sequence numbers
    * available for retransmission from this stack.
    *
    */
   private Collection getRebroadcastList(Collection rtr)
   {
      ArrayList rebroadcastList = new ArrayList(rtr);
      rebroadcastList.retainAll(receivedMessagesQueue.keySet());
      rtr.removeAll(rebroadcastList);
      Collections.sort(rebroadcastList);
      return rebroadcastList;
   }

   /**
    * TokenTransimitter thread transmits the token to the next member
    * in the logical ring as well as it receives token from the previous
    * member in the ring. Smoothed ring roundtrip time is computed
    * in order to detect token loss.  If the timeout expires AND this
    * stack has received SUSPECT message, recovery protocol is invoked.
    * See recover method for details.
    *
    */
   private class TokenTransmitter extends Thread
   {
      long rtt = 0;
      long timer;
      double srtt = 1000; //1 second to start
      final double a = 0.09;
      final int timeoutFactor = 10;
      volatile boolean running = false;

      private TokenTransmitter()
      {
         super(Util.getGlobalThreadGroup(), "TokenTransmitter");
         resetTimeout();
         running = true;
      }

      private void shutDown()
      {
          running = false;
      }

      private void recalculateTimeout()
      {
         long now = System.currentTimeMillis();
         if (timer > 0)
         {
            rtt = now - timer;
            srtt = (1 - a) * srtt + a * rtt;
         }
      }

      private double getTimeout()
      {
         return srtt * timeoutFactor;
      }

      private void resetTimeout()
      {
         timer = System.currentTimeMillis();
      }

      private boolean isRecoveryCompleted(RingToken token)
      {
          return liveMembersInRecovery.equals(token.getRecoveredMembers());
      }

      public void run()
      {
         while (running)
         {
            RingToken token = null;
            int timeout = 0;

             if(node == null) {
                 // sleep some time, then retry
                 Util.sleep(500);
                 continue;
             }

            try
            {
               timeout = (int) getTimeout();

                  if(log.isInfoEnabled()) log.info("timeout(ms)=" + timeout);

               token = (RingToken) node.receiveToken(timeout);

               if (token.getType() == RingToken.OPERATIONAL &&
                       state == RECOVERY_STATE)
               {
                  state = OPERATIONAL_STATE;
               }

               tokenReceived(token);
               recalculateTimeout();

               if (token.getType() == RingToken.RECOVERY &&
                       isRecoveryCompleted(token))
               {

                     if(log.isInfoEnabled()) log.info("all members recovered, injecting operational token");
                  token.setType(RingToken.OPERATIONAL);
               }
               node.passToken(token);
               resetTimeout();
            }
            catch (TokenLostException tle)
            {
               invalidateOnTokenloss();
               state = RECOVERY_STATE;
               recover();
            }
         }
      }
   }
}

