/*
 * JBoss, the OpenSource J2EE webOS
 *
 * Distributable under LGPL license.
 * See terms of license at gnu.org.
 */

package org.jboss.cache.aop.statetransfer;

import EDU.oswego.cs.dl.util.concurrent.Semaphore;
import junit.framework.TestCase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jboss.cache.CacheException;
import org.jboss.cache.Fqn;
import org.jboss.cache.PropertyConfigurator;
import org.jboss.cache.TreeCache;
import org.jboss.cache.aop.PojoCache;
import org.jboss.cache.aop.test.Address;
import org.jboss.cache.aop.test.Person;
import org.jboss.cache.loader.CacheLoader;
import org.jboss.cache.misc.TestingUtil;
import org.jboss.cache.xml.XmlHelper;
import org.w3c.dom.Element;

import javax.transaction.TransactionManager;
import java.io.File;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Random;
import java.util.Set;

/**
 * Tests state transfer in PojoCache.
 * 
 * @author <a href="mailto://brian.stansberry@jboss.com">Brian Stansberry</a>
 * @version $Revision$
 */
public abstract class StateTransferAopTestBase extends TestCase
{
   private Map caches;

   public static final Fqn A_B_1 = Fqn.fromString("/a/b/1");
   public static final Fqn A_B_2 = Fqn.fromString("/a/b/2");
   public static final Fqn A_C_1 = Fqn.fromString("/a/c/1");
   public static final Fqn A_C_2 = Fqn.fromString("/a/c/2");

   private static final int SUBTREE_SIZE = 10;
   
   private Person  joe;
   private Person  bob;
   private Person  jane;
   private Person  jill;
   private Address addr1;
   private Address addr2;

   public static final Integer TWENTY = new Integer(20);
   public static final Integer TWENTYFIVE = new Integer(25);
   public static final Integer FORTY = new Integer(40);
   
   private Log log=LogFactory.getLog(StateTransferAopTestBase.class);
   
   public void testInitialStateTransfer() throws Exception
   {
      log.info("Enter testInitialStateTransfer");
      
      PojoCache cache1 = createCache("cache1", false, false, false);

      cache1.putObject(A_B_1, joe);
      cache1.putObject(A_B_2, jane);
      cache1.putObject(A_C_1, bob);
      cache1.putObject(A_C_2, jill);
      
      PojoCache cache2 = createCache("cache2", false, false, false);

      // Pause to give caches time to see each other
      TestingUtil.blockUntilViewsReceived(new PojoCache[]
      {cache1, cache2}, 60000);

      Person ab1  = (Person) cache2.getObject(A_B_1);
      Person ab2 = (Person) cache2.getObject(A_B_2);
      Person ac1  = (Person) cache2.getObject(A_C_1);
      Person ac2 = (Person) cache2.getObject(A_C_2);
      assertEquals("Name for /a/b/1 is Joe", joe.getName(), ab1.getName());
      assertEquals("City for /a/b/1 is Anytown", addr1.getCity(), ab1.getAddress().getCity());
      assertEquals("Name for /a/b/2 is Jane", jane.getName(), ab2.getName());
      assertEquals("City for /a/b/2 is Anytown", addr1.getCity(), ab2.getAddress().getCity());
      assertTrue("Joe and Jane have same Address", ab1.getAddress() == ab2.getAddress());
      assertEquals("Name for /a/c/1 is Bob", bob.getName(), ac1.getName());
      assertEquals("City for /a/c/1 is Fremont", addr2.getCity(), ac1.getAddress().getCity());
      assertEquals("Name for /a/c/2 is Jill", jill.getName(), ac2.getName());
      assertEquals("City for /a/c/2 is Fremont", addr2.getCity(), ac2.getAddress().getCity());
      assertTrue("Bob and Jill have same Address", ac1.getAddress() == ac2.getAddress());
   }

   public void testInitialStateTferWithLoader() throws Exception
   { 
      log.info("Enter testInitialStateTferWithLoader");
      
      PojoCache cache1 = createCache("cache1", false, false, true);

      cache1.putObject(A_B_1, joe);
      cache1.putObject(A_B_2, jane);
      cache1.putObject(A_C_1, bob);
      cache1.putObject(A_C_2, jill);

      PojoCache cache2 = createCache("cache2", false, false, true);

      // Pause to give caches time to see each other
      TestingUtil.blockUntilViewsReceived(new PojoCache[]
      {cache1, cache2}, 60000);

      CacheLoader loader = cache2.getCacheLoader();

      assertEquals("Loader name for /a/b/1 is Joe", joe.getName(), loader.get(A_B_1).get("name"));
      assertEquals("Loader name for /a/b/2 is Jane", jane.getName(), loader.get(A_B_2).get("name"));
      assertEquals("Loader name for /a/c/1 is Joe", bob.getName(), loader.get(A_C_1).get("name"));
      assertEquals("Loader name for /a/c/2 is Jill", jill.getName(), loader.get(A_C_2).get("name"));
      
      Person ab1  = (Person) cache2.getObject(A_B_1);
      Person ab2 = (Person) cache2.getObject(A_B_2);
      Person ac1  = (Person) cache2.getObject(A_C_1);
      Person ac2 = (Person) cache2.getObject(A_C_2);
      assertEquals("Name for /a/b/1 is Joe", joe.getName(), ab1.getName());
      assertEquals("City for /a/b/1 is Anytown", addr1.getCity(), ab1.getAddress().getCity());
      assertEquals("Name for /a/b/2 is Jane", jane.getName(), ab2.getName());
      assertEquals("City for /a/b/2 is Anytown", addr1.getCity(), ab2.getAddress().getCity());
      assertTrue("Joe and Jane have same Address", ab1.getAddress() == ab2.getAddress());
      assertEquals("Name for /a/c/1 is Bob", bob.getName(), ac1.getName());
      assertEquals("City for /a/c/1 is Fremont", addr2.getCity(), ac1.getAddress().getCity());
      assertEquals("Name for /a/c/2 is Jill", jill.getName(), ac2.getName());
      assertEquals("City for /a/c/2 is Fremont", addr2.getCity(), ac2.getAddress().getCity());
      assertTrue("Bob and Jill have same Address", ac1.getAddress() == ac2.getAddress());
   }

   public void testPartialStateTransfer() throws Exception
   { 
      log.info("Enter testPartialStateTransfer");
      
      PojoCache cache1 = createCache("cache1", false, true, false);
      cache1.activateRegion("/a");

      cache1.putObject(A_B_1, joe);
      cache1.putObject(A_B_2, jane);

      PojoCache cache2 = createCache("cache2", false, true, false);

      // Pause to give caches time to see each other
      TestingUtil.blockUntilViewsReceived(new PojoCache[]
      {cache1, cache2}, 60000);
      
      assertNull("/a/b/1 not transferred per policy", cache2.getObject(A_B_1));
      assertNull("/a/b/2 not transferred per policy", cache2.getObject(A_B_2));

      cache2.activateRegion("/a/b");
      
      Person ab1  = (Person) cache2.getObject(A_B_1);
      Person ab2 = (Person) cache2.getObject(A_B_2);
      assertEquals("Name for /a/b/1 is Joe", joe.getName(), ab1.getName());
      assertEquals("City for /a/b/1 is Anytown", joe.getAddress().getCity(), ab1.getAddress().getCity());
      assertEquals("Name for /a/b/2 is Jane", jane.getName(), ab2.getName());
      assertEquals("City for /a/b/2 is Anytown", jane.getAddress().getCity(), ab2.getAddress().getCity());      
      assertTrue("Address for Joe and Jane is the same object", ab1.getAddress() == ab2.getAddress());

      cache1.putObject(A_C_1, bob);
      cache1.putObject(A_C_2, jill);
      
      assertNull("/a/c/1 not transferred per policy", cache2.getObject(A_C_1));

      cache2.activateRegion("/a/c");
      
      Person ac1 = (Person) cache2.getObject(A_C_1);
      Person ac2 = (Person) cache2.getObject(A_C_2);
      assertEquals("Name for /a/c/1 is Bob", bob.getName(), ac1.getName());
      assertEquals("City for /a/c/1 is Fremont", addr2.getCity(), ac1.getAddress().getCity());
      assertEquals("Name for /a/c/2 is Jill", jill.getName(), ac2.getName());
      assertEquals("City for /a/c/2 is Fremont", addr2.getCity(), ac2.getAddress().getCity());
      assertTrue("Bob and Jill have same Address", ac1.getAddress() == ac2.getAddress());

      cache1.inactivateRegion("/a");

      cache1.activateRegion("/a/b/1");
      cache1.activateRegion("/a/b/2");
      cache1.activateRegion("/a/c/1");
      cache1.activateRegion("/a/c/2");

      ab1  = (Person) cache1.getObject(A_B_1);
      assertEquals("Name for /a/b/1 is Joe", joe.getName(), ab1.getName());
      assertEquals("City for /a/b/1 is Anytown", addr1.getCity(), ab1.getAddress().getCity());
      ab2 = (Person) cache1.getObject(A_B_2);;
      assertEquals("Name for /a/b/1 is Jane", jane.getName(), ab2.getName());
      assertEquals("City for /a/b/1 is Anytown", addr1.getCity(), ab2.getAddress().getCity());
      assertTrue("Address for Joe and Jane is the same object", ab1.getAddress() == ab2.getAddress());
      ac1 = (Person) cache1.getObject(A_C_1);
      assertEquals("Name for /a/c/1 is Bob", bob.getName(), ac1.getName());
      assertEquals("City for /a/c/1 is Fremont", addr2.getCity(), ac1.getAddress().getCity());
      ac2 = (Person) cache1.getObject(A_C_2);
      assertEquals("Name for /a/c/2 is Jill", jill.getName(), ac2.getName());
      assertEquals("City for /a/c/2 is Fremont", addr2.getCity(), ac2.getAddress().getCity());
      assertTrue("Address for Bob and Jill is the same object", ac1.getAddress() == ac2.getAddress());
   }

   public void testPartialStateTransferWithLoader() throws Exception
   {  
      log.info("Enter testPartialStateTransferWithLoader");
      
      PojoCache cache1 = createCache("cache1", false, true, true);
      cache1.activateRegion("/a");

      cache1.putObject(A_B_1, joe);
      cache1.putObject(A_B_2, jane);

      PojoCache cache2 = createCache("cache2", false, true, true);

      // Pause to give caches time to see each other
      TestingUtil.blockUntilViewsReceived(new PojoCache[]
      {cache1, cache2}, 60000);
      
      CacheLoader loader = cache2.getCacheLoader();
      
      Map map = loader.get(A_B_1);
      if (map != null)
      {
         assertNull("/a/b/1 name not transferred per policy", map.get("name"));
         assertNull("/a/b/1 age not transferred per policy", map.get("age"));
      }
      map = loader.get(A_B_2);
      if (map != null)
      {
         assertNull("/a/b/1 name not transferred per policy", map.get("name"));
         assertNull("/a/b/1 age not transferred per policy", map.get("age"));
      }
      assertNull("/a/b/1 not transferred per policy", cache2.getObject(A_B_1));
      assertNull("/a/b/2 not transferred per policy", cache2.getObject(A_B_2));

      cache2.activateRegion("/a/b");

      assertEquals("Correct name from loader for /a/b/1", joe.getName(), loader.get(A_B_1).get("name"));
      assertEquals("Correct age from loader for /a/b/1", TWENTY, loader.get(A_B_1).get("age"));
      assertEquals("Correct name from loader for /a/b/2", jane.getName(), loader.get(A_B_2).get("name"));
      assertEquals("Correct age from loader for /a/b/2", TWENTYFIVE, loader.get(A_B_2).get("age"));
      
      
      Person ab1  = (Person) cache2.getObject(A_B_1);
      Person ab2 = (Person) cache2.getObject(A_B_2);
      assertEquals("Name for /a/b/1 is Joe", joe.getName(), ab1.getName());
      assertEquals("City for /a/b/1 is Anytown", addr1.getCity(), ab1.getAddress().getCity());
      assertEquals("Name for /a/b/1 is Jane", jane.getName(), ab2.getName());
      assertEquals("City for /a/b/1 is Anytown", addr1.getCity(), ab2.getAddress().getCity());
      assertTrue("Address for Joe and Jane is the same object", ab1.getAddress() == ab2.getAddress());

      cache1.putObject(A_C_1, bob);
      cache1.putObject(A_C_2, jill);
      
      assertNull("/a/c/1 not transferred per policy", cache2.getObject(A_C_1));

      cache2.activateRegion("/a/c");

      assertEquals("Correct name from loader for /a/b/1", joe.getName(), loader.get(A_B_1).get("name"));
      assertEquals("Correct age from loader for /a/b/1", TWENTY, loader.get(A_B_1).get("age"));
      assertEquals("Correct name from loader for /a/b/2", jane.getName(), loader.get(A_B_2).get("name"));
      assertEquals("Correct age from loader for /a/b/2", TWENTYFIVE, loader.get(A_B_2).get("age"));
      assertEquals("Correct name from loader for /a/c/1", bob.getName(), loader.get(A_C_1).get("name"));
      assertEquals("Correct age from loader for /a/c/1", FORTY, loader.get(A_C_1).get("age"));
      assertEquals("Correct name from loader for /a/c/2", jill.getName(), loader.get(A_C_2).get("name"));
      assertEquals("Correct age from loader for /a/c/2", TWENTYFIVE, loader.get(A_C_2).get("age"));
      
      Person ac1 = (Person) cache2.getObject(A_C_1);
      Person ac2 = (Person) cache2.getObject(A_C_2);
      assertEquals("Name for /a/c/1 is Bob", bob.getName(), ac1.getName());
      assertEquals("City for /a/c/1 is Fremont", addr2.getCity(), ac1.getAddress().getCity());
      assertEquals("Name for /a/c/2 is Jill", jill.getName(), ac2.getName());
      assertEquals("City for /a/c/2 is Fremont", addr2.getCity(), ac2.getAddress().getCity());
      assertTrue("Bob and Jill have same Address", ac1.getAddress() == ac2.getAddress());

      cache1.inactivateRegion("/a");

      cache1.activateRegion("/a/b/1");
      cache1.activateRegion("/a/b/2");
      cache1.activateRegion("/a/c/1");
      cache1.activateRegion("/a/c/2");

      ab1  = (Person) cache1.getObject(A_B_1);
      assertEquals("Name for /a/b/1 is Joe", joe.getName(), ab1.getName());
      assertEquals("City for /a/b/1 is Anytown", addr1.getCity(), ab1.getAddress().getCity());
      ab2 = (Person) cache1.getObject(A_B_2);;
      assertEquals("Name for /a/b/1 is Jane", jane.getName(), ab2.getName());
      assertEquals("City for /a/b/1 is Anytown", addr1.getCity(), ab2.getAddress().getCity());
      assertTrue("Address for Joe and Jane is the same object", ab1.getAddress() == ab2.getAddress());
      ac1 = (Person) cache1.getObject(A_C_1);
      assertEquals("Name for /a/c/1 is Bob", bob.getName(), ac1.getName());
      assertEquals("City for /a/c/1 is Fremont", addr2.getCity(), ac1.getAddress().getCity());
      ac2 = (Person) cache1.getObject(A_C_2);
      assertEquals("Name for /a/c/2 is Jill", jill.getName(), ac2.getName());
      assertEquals("City for /a/c/2 is Fremont", addr2.getCity(), ac2.getAddress().getCity());
      assertTrue("Address for Bob and Jill is the same object", ac1.getAddress() == ac2.getAddress());
   }


   
   /**
    * Tests concurrent activation of the same subtree by multiple nodes in a 
    * REPL_SYNC environment.  The idea is to see what would happen with a 
    * farmed deployment. See <code>concurrentActivationTest</code> for details.
    * 
    * @throws Exception
    */

   // commented out since concurrent activations will cause deadlocks and then timeouts since JBCACHE-875.
   // Correctness over performance. - Manik
   
//   public void testConcurrentActivationSync() throws Exception
//   {
//      log.info("Enter testConcurrentActivationSync");
//
//      concurrentActivationTest(true);
//   }

   /**
    * Tests concurrent activation of the same subtree by multiple nodes in a 
    * REPL_ASYNC environment.  The idea is to see what would happen with a 
    * farmed deployment. See <code>concurrentActivationTest</code> for details.
    * 
    * @throws Exception
    */
   public void testConcurrentActivationAsync() throws Exception
   { 
      log.info("Enter testConcurrentActivationAsync");
      
      concurrentActivationTest(false);
   }
   
   /**
    * Starts 5 caches and then concurrently activates the same region under
    * all 5, causing each to attempt a partial state transfer from the others.
    * As soon as each cache has activated its region, it does a put to a node
    * in the region, thus complicating the lives of the other caches trying
    * to get partial state.
    * <p>
    * Failure condition is if any node sees an exception or if the final state
    * of all caches is not consistent.
    * 
    * @param sync use REPL_SYNC or REPL_ASYNC
    * @throws Exception
    */
   private void concurrentActivationTest(boolean sync) throws Exception
   {
      String[] names = { "A", "B", "C", "D", "E" };
      int count = names.length;
      CacheActivator[] activators = new CacheActivator[count];
      
      
      try {
         // Create a semaphore and take all its tickets
         Semaphore semaphore = new Semaphore(count);         
         for (int i = 0; i < count; i++) {
            semaphore.acquire();
         }   
         
         // Create activation threads that will block on the semaphore
         TreeCache[] caches = new TreeCache[count];
         for (int i = 0; i < count; i++) {
            activators[i] = new CacheActivator(semaphore, names[i], sync);
            caches[i] = activators[i].getTreeCache();
            activators[i].start();
         }
         
         // Make sure everyone is in sync
         TestingUtil.blockUntilViewsReceived(caches, 60000);
         
         // Release the semaphore to allow the threads to start work 
         semaphore.release(count);
         
         // Sleep to ensure the threads get all the semaphore tickets
         TestingUtil.sleepThread(1000);
         
         // Reacquire the semaphore tickets; when we have them all
         // we know the threads are done         
         for (int i = 0; i < count; i++) {
            boolean acquired = semaphore.attempt(60000);
            if (!acquired)
               fail("failed to acquire semaphore " + i);
         }

         // Sleep to allow any async calls to clear
         if (!sync)
            TestingUtil.sleepThread(500);
         
         // Ensure the caches held by the activators see all the values
         for (int i = 0; i < count; i++)
         {
            assertNull("Activator " + names[i] + " caught an exception", 
                       activators[i].getException());
            
            for (int j = 0; j < count; j++)
            {
               String fqn = "/a/b/" + names[j];
               Person p = (Person) activators[i].getCacheValue(fqn);
               assertNotNull(names[i] + ":" + fqn + " is not null", p);
               assertEquals("Correct name for " + names[i] + ":" + fqn, 
                             "Person " + names[j], p.getName());
               assertEquals("Correct street for " + names[i] + ":" + fqn, 
                     names[j] + " Test Street", p.getAddress().getStreet());
//               System.out.println(names[i] + ":" + fqn + " = " + activators[i].getCacheValue(fqn));
            }
            
         }
      }
      catch (Exception ex) {
         fail(ex.getLocalizedMessage());
      }
      finally {
         for (int i = 0; i < count; i++)
            activators[i].cleanup();
      }
      
   }
   /**
    * Tests partial state transfer under heavy concurrent load and REPL_SYNC.
    * See <code>concurrentUseTest</code> for details.
    * 
    * @throws Exception
    */

   // commented out since concurrent activations will cause deadlocks and then timeouts since JBCACHE-875.
   // Correctness over performance. - Manik

//   public void testConcurrentUseSync() throws Exception
//   {
//      log.info("Enter testConcurrentUseSync");
//
//      concurrentUseTest(true);
//   }
//
   /**
    * Tests partial state transfer under heavy concurrent load and REPL_ASYNC.
    * See <code>concurrentUseTest</code> for details.
    * 
    * @throws Exception
    */
   public void testConcurrentUseAsync() throws Exception
   { 
      log.info("Enter testConcurrentUseAsync");
      
      concurrentUseTest(false);
   }
   
   /**
    * Initiates 5 caches, 4 with active trees and one with an inactive tree.
    * Each of the active caches begins rapidly generating puts against nodes 
    * in a subtree for which it is responsible. The 5th cache activates
    * each subtree, and at the end confirms no node saw any exceptions and
    * that each node has consistent state.
    * 
    * @param sync whether to use REPL_SYNC or REPL_ASYNCE
    * 
    * @throws Exception
    */
   private void concurrentUseTest(boolean sync) throws Exception
   {
      String[] names = { "B", "C", "D", "E" };
      int count = names.length;
      CacheStressor[] stressors = new CacheStressor[count];
      
      try {
         
         PojoCache cacheA = createCache("cacheA", sync, true, false, false);
         
         PojoCache[] caches = new PojoCache[count + 1];
         caches[0] = cacheA;
         
         // Create a semaphore and take all its tickets
         Semaphore semaphore = new Semaphore(count);         
         for (int i = 0; i < count; i++) {
            semaphore.acquire();
         }   
         
         // Create stressor threads that will block on the semaphore
                  
         for (int i = 0; i < count; i++) 
         {
            stressors[i] = new CacheStressor(semaphore, names[i], sync);
            caches[i + 1] = stressors[i].getTreeCache();
            stressors[i].getTreeCache().put("/" + names[i], null);
         }
         for (int i = 0; i < count; i++)
         {
            stressors[i].start();
            // Give each one a chance to stabilize
            TestingUtil.sleepThread(100);
         }
         
         // Make sure everyone's views are in sync
         TestingUtil.blockUntilViewsReceived(caches, 60000);
         
         // Repeat the basic test two times in order to involve inactivation
         for (int x = 0; x < 2; x++)
         {
//            if (x > 0)
//            {
               // Reset things by inactivating the region 
               // and enabling the stressors
               for (int i = 0; i < count; i++)
               {
                  cacheA.inactivateRegion("/" + names[i]);
                  log.info("TEST: Run " + x + "-- /" + names[i] + " inactivated on A");
                  stressors[i].startPuts();
               }
//            }
            
            // Release the semaphore to allow the threads to start work 
            semaphore.release(count);
            
            // Sleep to ensure the threads get all the semaphore tickets
            // and to ensure puts are actively in progress
            TestingUtil.sleepThread(300);
            
            // Activate cacheA
            for (int i = 0; i < count; i++)
            {
               log.info("TEST: Activating /" + names[i] + " on A");
               cacheA.activateRegion("/" + names[i]);
               // Stop the stressor so we don't pollute cacheA's state
               // with too many messages sent after activation -- we want
               // to compare transferred state with the sender
               stressors[i].stopPuts();
               log.info("TEST: Run " + x + "-- /" + names[i] + " activated on A");
               // Reacquire one semaphore ticket
               boolean acquired = semaphore.attempt(60000);
               if (!acquired)
                  fail("failed to acquire semaphore " + names[i]);
               log.info("TEST: Run " + x + "-- acquired semaphore from " + names[i]);
               
               // Pause to allow other work to proceed
               TestingUtil.sleepThread(100);
            }
            
            // Sleep to allow any in transit msgs to clear
            if (!sync)
               TestingUtil.sleepThread(2000);
            
            // Ensure the stressors saw no exceptions
            for (int i = 0; i < count; i++)
            {
               Exception e = stressors[i].getException();
               if (e != null)
               {
                  log.error("Stressor " + names[i] + " caught an exception", 
                          e);
                  throw e;
               }
            }
            
            log.info("Cache A details:\n" + cacheA.printDetails());
            
            // Compare cache contents
            Person p1 = null;
            Person p2 = null;
            for (int i = 0; i < count; i++)
            {
               log.info("Cache " + names[i] + " details:\n" +
                        stressors[i].getTreeCache().printDetails());
               
               for (int j = 0; j < SUBTREE_SIZE; j++)
               {
                  
                  String fqn = "/" + names[i] +"/" + j;
                  log.info("TEST: Getting A:" + fqn);
                  p1 = (Person) cacheA.getObject(fqn);
                  boolean p1Null = p1 == null;
                  log.info("TEST: Getting " + names[i] + ":" + fqn);
                  p2 = (Person) stressors[i].getTreeCache().getObject(fqn);
                  boolean p2Null = p2 == null;
                  assertEquals("Run " + x + ": " + fqn + 
                               " null status matches", p1Null, p2Null);
                  if (!p1Null)
                  {
                     assertEquals("Run " + x + ": A:" + fqn + " age matches " + names[i] + ":" + fqn,
                                  p1.getAge(), p2.getAge());
                     assertEquals("Run " + x + ": A:" + fqn + " name matches " + names[i] + ":" + fqn,
                                  p1.getName(), p2.getName());
                     assertEquals("Run " + x + ": A:" + fqn + " address matches " + names[i] + ":" + fqn,
                                  p1.getAddress().getStreet(), 
                                  p2.getAddress().getStreet());
                  }
               }
            }
         }
         
         for (int i = 0; i < count; i++)
            stressors[i].stopThread();
         
      }
      finally {
         for (int i = 0; i < count; i++)
         {
            if (stressors[i] != null)
               stressors[i].cleanup();
         }
      }
      
   }

   protected PojoCache createCache(String cacheID, boolean sync, boolean useMarshalling, boolean useCacheLoader)
         throws Exception
   {
      return createCache(cacheID, sync, useMarshalling, useCacheLoader, true);
   }
   
   protected PojoCache createCache(String cacheID, boolean sync,
                                      boolean useMarshalling, 
                                      boolean useCacheLoader,
                                      boolean inactiveOnStartup)
      throws Exception
   {
      if (caches.get(cacheID) != null)
         throw new IllegalStateException(cacheID + " already created");

      PojoCache tree = new PojoCache();
      PropertyConfigurator config = new PropertyConfigurator();
      String configFile = sync ? "META-INF/replSync-service.xml" : "META-INF/replAsync-service.xml";
      config.configure(tree, configFile); // read in generic replAsync xml
      //tree.setDeadlockDetection(sync);
      tree.setClusterName("StateTransferTestBase");
      tree.setReplicationVersion(getReplicationVersion());
      // Use a long timeout to facilitate setting debugger breakpoints
      tree.setInitialStateRetrievalTimeout(60000);
      if (useMarshalling)
      {
         tree.setUseMarshalling(true);
         tree.setInactiveOnStartup(inactiveOnStartup);
      }
      if (useCacheLoader)
      {
         configureCacheLoader(tree, cacheID, useMarshalling);
      }
      
      // Put the cache in the map before starting, so if it fails in
      // start it can still be destroyed later
      caches.put(cacheID, tree);
      
      tree.createService();
      tree.startService();

      return tree;
   }

   protected void configureCacheLoader(PojoCache cache, String cacheID, boolean useExtended) throws Exception
   {
      String tmp_location = getTempLocation(cacheID);

      // Do cleanup in case it failed before
      File file = new File(tmp_location);
      cleanFile(file);

      file.mkdir();

      tmp_location = escapeWindowsPath(tmp_location);
      
      if (useExtended)
          cache.setCacheLoaderConfiguration(getCacheLoaderConfig("org.jboss.cache.loader.FileExtendedCacheLoader", tmp_location));
      else
          cache.setCacheLoaderConfiguration(getCacheLoaderConfig("org.jboss.cache.loader.FileCacheLoader", tmp_location));
   }


    protected Element getCacheLoaderConfig(String cl, String loc) throws Exception
    {
        String xml = "            <config>\n" +
                "                \n" +
                "                <passivation>false</passivation>\n" +
                "                <preload></preload>\n" +
                "\n" +
                "                <cacheloader>\n" +
                "                    <class>"+cl+"</class>\n" +
                "                    <properties>\n" +
                "                        location=" + loc + "\n" +
                "                    </properties>\n" +
                "                    <async>false</async>\n" +
                "                    <fetchPersistentState>true</fetchPersistentState>\n" +
                "                    <ignoreModifications>false</ignoreModifications>\n" +
                "                </cacheloader>\n" +
                "                \n" +
                "            </config>";
        return XmlHelper.stringToElement(xml);
    }

   protected String getTempLocation(String cacheID)
   {
      String tmp_location = System.getProperty("java.io.tmpdir", "c:\\tmp");
      File file = new File(tmp_location);
      file = new File(file, cacheID);
      return file.getAbsolutePath();
   }

   protected String escapeWindowsPath(String path)
   {
      if ('/' == File.separatorChar)
         return path;
      
      char[] chars = path.toCharArray();
      StringBuffer sb = new StringBuffer();
      for (int i = 0; i < chars.length; i++)
      {
         if (chars[i] == '\\')
            sb.append('\\');
         sb.append(chars[i]);
      }
      return sb.toString();
   }
   
   protected abstract String getReplicationVersion();

   protected void setUp() throws Exception
   {
      super.setUp();

      caches = new HashMap();
      
      addr1 = new Address();
      addr1.setStreet("101 Oakview Dr");
      addr1.setCity("Anytown");
      addr1.setZip(11111);
      
      addr2 = new Address();
      addr2.setStreet("222 Happy Dr");
      addr2.setCity("Fremont");
      addr2.setZip(22222);
      
      joe = new Person();
      joe.setName("Joe");
      joe.setAge(TWENTY.intValue());
      joe.setAddress(addr1);
      Set skills = new HashSet();
      skills.add("TENNIS");
      skills.add("CARPENTRY");
      joe.setSkills(skills);

      jane = new Person();
      jane.setName("Jane");
      jane.setAge(TWENTYFIVE.intValue());
      jane.setAddress(addr1);
      skills = new HashSet();
      skills.add("JUJITSU");
      skills.add("MACRAME");
      jane.setSkills(skills);

      bob = new Person();
      bob.setName("Bob");
      bob.setAge(FORTY.intValue());
      bob.setAddress(addr2);
      skills = new HashSet();
      skills.add("LANGUAGES");
      skills.add("LAWN BOWLING");
      bob.setSkills(skills);

      jill = new Person();
      jill.setName("Jill");
      jill.setAge(TWENTYFIVE.intValue());
      jill.setAddress(addr2);
      skills = new HashSet();
      skills.add("FORTRAN");
      skills.add("COBOL");
      jane.setSkills(skills);
   }

   protected void tearDown() throws Exception
   {
      super.tearDown();

      Set keys = caches.keySet();
      String[] cacheIDs = new String[keys.size()];
      cacheIDs = (String[]) keys.toArray(cacheIDs);
      for (int i = 0; i < cacheIDs.length; i++)
      {
         stopCache((PojoCache) caches.get(cacheIDs[i]));
         File file = new File(getTempLocation(cacheIDs[i]));
         cleanFile(file);
      }
   }

   protected void stopCache(PojoCache cache)
   {
      if (cache != null)
      {
         try
         {
            TransactionManager m = cache.getTransactionManager();
            try
            {
               if (m!= null && m.getTransaction() != null)
               {
                  m.rollback();
               }
            }
            catch (Exception e)
            {
            }
            cache.stopService();
            cache.destroyService();
         }
         catch (Exception e)
         {
            log.error("Exception stopping cache " + e.getMessage(), e);
         }
      }
   }

   protected void cleanFile(File file)
   {
      File[] children = file.listFiles();
      if (children != null)
      {
         for (int i = 0; i < children.length; i++)
         {
            cleanFile(children[i]);
         }
      }

      if (file.exists())
         file.delete();
      if (file.exists())
         file.deleteOnExit();
   }
   
   private class CacheActivator extends CacheUser
   {
      
      CacheActivator(Semaphore semaphore, 
                     String name,
                     boolean sync)
               throws Exception
      {
         super(semaphore, name, sync, false);
      }
      
      void useCache() throws Exception
      {
         cache.activateRegion("/a/b");
         log.info("TEST: " + name + " activated region" + " " + System.currentTimeMillis());
         String childFqn = "/a/b/" + name;
         
         Person p = new Person();
         p.setName("Person " + name);
         
         Address addr = new Address();         
         addr.setStreet(name + " Test Street");
         addr.setCity(name + ", CA");
         p.setAddress(addr);
         
         TestingUtil.sleepThread(1);
         
//         tm.begin();
//         try
//         {
            cache.putObject(childFqn, p);
            log.info("TEST: " + name + " put fqn " + childFqn + " " + System.currentTimeMillis());
//         }
//         catch (Exception e)
//         {
//            tm.setRollbackOnly();
//            throw e;
//         }
//         finally
//         {
//            tm.commit();
//         }

      }
      
      public Object getCacheValue(String fqn) throws CacheException
      {
         return cache.getObject(fqn);
      }
   }
   
   private class CacheStressor extends CacheUser
   {
      private Random random;
      private boolean putsStopped = false;
      private boolean stopped     = false;
      
      CacheStressor(Semaphore semaphore, 
                     String name,
                     boolean sync)
         throws Exception
      {
         super(semaphore, name, sync, true);
         
         random = new Random(System.currentTimeMillis() + name.hashCode());
      }

      void useCache() throws Exception
      {
         // Do lots of puts into the cache.  Use our own nodes,
         // as we're not testing conflicts between writer nodes,
         // just whether activation causes problems
         int factor = 0;
         int i = 0;
         String fqn = null;
         
         Address addr1 = new Address();
         addr1.setStreet("1 Test Street");
         addr1.setCity("TestOne, CA");
         
         Address addr2 = new Address();
         addr2.setStreet("2 Test Street");
         addr2.setCity("TestTwo, CA");
         
         Person[] people = new Person[SUBTREE_SIZE];
         boolean[] loaded = new boolean[SUBTREE_SIZE];
         for (int j = 0; j < SUBTREE_SIZE; j++)
         {
            Person p = new Person();
            p.setName("Person " + j);
            p.setAge(j);
            p.setAddress((j % 2 == 0) ? addr1 : addr2);
            people[j] = p;
         }
         
         boolean acquired = true;
         try 
         {
            while (!stopped)
            {
               if (i > 0)
               {
                  acquired = semaphore.attempt(60000);
                  if (!acquired)
                     throw new Exception(name + " cannot acquire semaphore");
                  log.info("TEST: " + name + " reacquired semaphore");
                  System.out.println("TEST: " + name + " reacquired semaphore");
               }               
               
               int lastIndex = -1;
               int index = -1;
               while(!putsStopped)
               {
                  // Ensure we don't operate on the same address twice in a row
                  // otherwise deadlock detection sometimes causes
                  // the _put for the second call to precede the commit
                  // for the first, leading to deadlock.  This seems like a 
                  // JGroups bug, but the purpose of this test isn't to expose it
                  while (index % 2 == lastIndex % 2) {
                     factor = random.nextInt(50);
                     index = factor % SUBTREE_SIZE;
                  }
                  
                  lastIndex = index;
                  
                  TestingUtil.sleepThread(factor);
                  
                  fqn = "/" + name +"/" + String.valueOf(index);
                  
//                  tm.begin();
//                  try
//                  {
                     if (loaded[index] == false) {
                        cache.putObject(fqn, people[index]);
                        loaded[index] = true;
                        log.info("TEST: " + name + " put Person at " + fqn);
                     }
                     else if (i % 2 == 0){
                        int newAge = factor / SUBTREE_SIZE;
                        people[index].setAge(newAge);                  
                     }
                     else {
                        people[index].getAddress().setStreet(factor + " Test Street");
                     }
//                  }
//                  catch (Exception e)
//                  {
//                     tm.setRollbackOnly();
//                     throw e;
//                  }
//                  finally
//                  {
//                     tm.commit();
//                  }              
                  
                  i++;
               }
               
               log.info("TEST: " + name + ": last put [#"+ i + "] -- " + fqn + " = " + (factor / SUBTREE_SIZE));
               
               semaphore.release();
               acquired = false;
               
               // Go to sleep until directed otherwise
               while (!stopped && putsStopped)
                  TestingUtil.sleepThread(100);
            }
         }
         finally {
            if (acquired)
               semaphore.release();
         }
      }
      
      
      
//      public void start() throws Exception
//      {
//         super.start();
//         cache.activateRegion("/" + name);
//      }

      public void stopPuts()
      {
         putsStopped = true;
         log.info("TEST: " + name + " putsStopped");
      }
      
      public void startPuts()
      {
         putsStopped = false;
      }
      
      public void stopThread()
      {
         stopped = true;
         if (thread.isAlive())
            thread.interrupt();
      }
      
      
   }
   
   private abstract class CacheUser implements Runnable
   {
      protected Semaphore semaphore;
      protected PojoCache cache;
      protected TransactionManager tm;
      protected String name;
      protected Exception exception;
      protected Thread thread;
      
      CacheUser(Semaphore semaphore, 
                String name,
                boolean sync,
                boolean activateRoot)
            throws Exception
      {
         this.cache = createCache(name, sync, true, false, !activateRoot);
         tm = cache.getTransactionManager();
         if (tm == null)
            throw new IllegalStateException("TransactionManager required");
         this.semaphore = semaphore;
         this.name = name;
        
         log.info("TEST: Cache " + name + " started");
         System.out.println("TEST: Cache " + name + " started");
      }
      
      public void run()
      {
         log.info("TEST: " + name + " started");
         System.out.println("TEST: " + name + " started");
         
         boolean acquired = false;
         try
         {
            acquired = semaphore.attempt(60000);
            if (!acquired)
               throw new Exception(name + " cannot acquire semaphore");
            log.info("TEST: " + name + " acquired semaphore");
            System.out.println("TEST: " + name + " acquired semaphore");
            useCache();
            
         }
         catch (Exception e) 
         {
            log.error("TEST: " + name + ": " + e.getLocalizedMessage(), e);
            
            // Save it for the test to check
            exception = e;
         }
         finally 
         {
            if (acquired)
               semaphore.release();
         }
         
      }
      
      abstract void useCache() throws Exception;
      
      public Exception getException()
      {
         return exception;
      }
      
      public PojoCache getTreeCache()
      {
         return cache;
      }
      
      public void start() throws Exception
      {         
         thread = new Thread(this);
         thread.start();         
      }
      
      public void cleanup()
      {         
         if (thread != null && thread.isAlive())
            thread.interrupt();
      }
   }
}
