/*                          
 * mBench: The Open Source Micro Benchmark Tool 
 *                                             
 * Distributable under GPL license. 
 * See terms of license at gnu.org.                 
 *
 * Copyright (C) 2005 Sumisho Computer Systems Corp.
 */
package test.jp.co.scs.mbench;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import jp.co.scs.mbench.BenchmarkAbortException;
import jp.co.scs.mbench.BenchmarkContainer;
import jp.co.scs.mbench.BenchmarkInfo;
import jp.co.scs.mbench.BenchmarkMain;
import jp.co.scs.mbench.BenchmarkManager;
import jp.co.scs.mbench.BenchmarkState;
import jp.co.scs.mbench.BenchmarkTerminateException;
import jp.co.scs.mbench.DataWriter;
import jp.co.scs.mbench.LogWriter;
import jp.co.scs.mbench.Synchronizer;
import junit.framework.TestCase;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerComponent2;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerComponent3;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerContainer;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerContainer2;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerContainer3;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerDataWriter;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerDataWriter2;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerLogWriter;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerLogWriter2;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerMultiNode01;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerMultiNode02;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerMultiNode03;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerMultiNode04;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerMultiNode05;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerSecondaryNode;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerSecondaryNode2;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerSecondaryNode3;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerSecondaryNode4;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerSecondaryNode5;
import test.jp.co.scs.mbench.stub.StubBenchmarkManagerSecondaryNode6;

/**
 *      @author Toshio Sugahara
 *      @author Tetsuro Ikeda
 */
public class BenchmarkManagerTest extends TestCase {
    
    private static String BENCHMARK_NAME = "benchmark-name";
    
    private static String WRONG_BENCHMARK_CLASS_NAME =
        "test.jp.co.scs.mbench.stub.StubWrongBenchmarkManagerComponent";
    
    private static String BENCHMARK_CLASS_NAME =
        "test.jp.co.scs.mbench.stub.StubBenchmarkManagerComponent";
    
    private static String BENCHMARK_CLASS_NAME2 =
        "test.jp.co.scs.mbench.stub.StubBenchmarkManagerComponent2";
    
    private static String BENCHMARK_CLASS_NAME3 =
        "test.jp.co.scs.mbench.stub.StubBenchmarkManagerComponent3";
    
    private static int THREAD_NUMBER = 5;
    
    private static int REPEAT_NUMBER = 10;
    
    private static int TRANSACTION_NUMBER = 100;
    
    private static long END_TIME = 150;
    
    private static final String MULTI_NODE_NAME = "secondary_multi";
    
    private static final String MULTI_NODE_NAME2 = "secondary_multi2";
    
    private static final String MULTI_NODE_NAME3 = "secondary_multi3";
    
    private static final String MULTI_NODE_NAME4 = "secondary_multi4";
    
    private static final String MULTI_NODE_NAME5 = "secondary_multi5";
    
    private static final String MULTI_NODE_NAME6 = "secondary_multi6";
    
    private static final String WRONG_MULTI_NODE_NAME = "wrong_secondary_multi";
    
    private static String LOG_NAME = "log-name";
    
    private static final String CONFIGURATION_FILE_NAME = "conf/mbench.xml";
    
    private static final String TEST_CONFIGURATION_FILE_NAME1 = 
        "data/BenchmarkManager/SecondaryMultiNode/mbench.xml";
    
    private static final String DEFAULT_DIR = "conf";
    
    private static final String TEMP_DIR = "temp";

    /**
     *  The state must be "BENCHMARK_SUCCESS" if benchmark was succeeded.
     *  (without option map)
     */
    public void testRun() {
        BenchmarkManager manager = new  BenchmarkManager(BENCHMARK_NAME);
        
        BenchmarkInfo info = new BenchmarkInfo();
        manager.setBenchmarkInfo(info);
        info.setName(BENCHMARK_NAME);
        info.setClassName(BENCHMARK_CLASS_NAME);
        info.setThreadNumber(THREAD_NUMBER);
        info.setRepeatNumber(REPEAT_NUMBER);
        info.setTransactionNumber(TRANSACTION_NUMBER);
        info.setBenchmarkEndTime(END_TIME);
        
        LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
        manager.setLogWriter(stubLogWriter);
        DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
        manager.setDataWriter(stubDataWriter);
        
        Synchronizer sync = new Synchronizer();
        manager.setSynchronizer(sync);
        
        BenchmarkContainer[] containers = new BenchmarkContainer[info.getThreadNumber()];
        for (int i = 0; i < info.getThreadNumber(); i++) {
            int containerID = i;
            containers[i] = new StubBenchmarkManagerContainer(
                containerID, info, sync, stubLogWriter, stubDataWriter);
        }
        
        try {
            manager.run();
            assertEquals(BenchmarkState.BENCHMARK_SUCCESS, manager.getCurrentState());
        } catch (BenchmarkAbortException ex) {
            fail(ex.getMessage());
        }
    }

    /**
     *  The state must be "BENCHMARK_SUCCESS" if benchmark was succeeded.
     *  (with option map)
     *  The benchmark information must be overridden by the option map.
     */
    public void testRun2() {
        BenchmarkManager manager = new  BenchmarkManager(BENCHMARK_NAME);
        
        BenchmarkInfo info = new BenchmarkInfo();
        manager.setBenchmarkInfo(info);
        info.setName(BENCHMARK_NAME);
        info.setClassName(BENCHMARK_CLASS_NAME);
        info.setThreadNumber(THREAD_NUMBER);
        info.setRepeatNumber(REPEAT_NUMBER);
        info.setTransactionNumber(TRANSACTION_NUMBER);
        info.setBenchmarkEndTime(END_TIME);
        
        int newThreadNumber = 15;
        int newRepeatNumber = 20;
        int newTransactionNumber = 400;
        long newBenchmarkEndTime = 100;
        
        Map optionMap = new HashMap();
        manager.setOptionMap(optionMap);
        optionMap.put(BenchmarkMain.THREAD_NUMBER, String.valueOf(newThreadNumber));
        optionMap.put(BenchmarkMain.REPEAT_NUMBER, String.valueOf(newRepeatNumber));
        optionMap.put(BenchmarkMain.TX_NUMBER, String.valueOf(newTransactionNumber));
        optionMap.put(BenchmarkMain.END_TIME, String.valueOf(newBenchmarkEndTime));
        
        LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
        manager.setLogWriter(stubLogWriter);
        DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
        manager.setDataWriter(stubDataWriter);
        
        Synchronizer sync = new Synchronizer();
        manager.setSynchronizer(sync);
        
        BenchmarkContainer[] containers = new BenchmarkContainer[info.getThreadNumber()];
        for (int i = 0; i < info.getThreadNumber(); i++) {
            int containerID = i;
            containers[i] = new StubBenchmarkManagerContainer(
                containerID, info, sync, stubLogWriter, stubDataWriter);
        }
        
        try {
            manager.run();
            assertEquals(BenchmarkState.BENCHMARK_SUCCESS, manager.getCurrentState());
            assertTrue(THREAD_NUMBER != newThreadNumber);
            assertEquals(newThreadNumber, info.getThreadNumber());
            assertTrue(REPEAT_NUMBER != newRepeatNumber);
            assertEquals(newRepeatNumber, info.getRepeatNumber());
            assertTrue(TRANSACTION_NUMBER != newTransactionNumber);
            assertEquals(newTransactionNumber, info.getTransactionNumber());
            assertTrue(END_TIME != newBenchmarkEndTime);
            assertEquals(newBenchmarkEndTime, info.getBenchmarkEndTime());
            
        } catch (BenchmarkAbortException ex) {
            fail(ex.getMessage());
        }
    }

    /**
     *  The BenchmarkAbortException must be thrown if LogWriter couldn't 
     *  be initialized. In this case, the state must be "BENCHMARK_FAIL".
     */
    public void testRun3() {
        BenchmarkManager manager = new  BenchmarkManager(BENCHMARK_NAME);
        
        BenchmarkInfo info = new BenchmarkInfo();
        manager.setBenchmarkInfo(info);
        info.setName(BENCHMARK_NAME);
        info.setClassName(BENCHMARK_CLASS_NAME);
        info.setThreadNumber(THREAD_NUMBER);
        info.setRepeatNumber(REPEAT_NUMBER);
        
        LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter2(LOG_NAME);
        manager.setLogWriter(stubLogWriter);
        DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
        manager.setDataWriter(stubDataWriter);
        
        Synchronizer sync = new Synchronizer();
        manager.setSynchronizer(sync);
        
        BenchmarkContainer[] containers = new BenchmarkContainer[info.getThreadNumber()];
        for (int i = 0; i < info.getThreadNumber(); i++) {
            int containerID = i;
            containers[i] = new StubBenchmarkManagerContainer(
                containerID, info, sync, stubLogWriter, stubDataWriter);
        }
        
        try {
            manager.run();
            fail();
        } catch (BenchmarkAbortException ex) {
            assertEquals(BenchmarkState.BENCHMARK_FAIL, manager.getCurrentState());
        }
    }

    /**
     *  The BenchmarkAbortException must be thrown if LogWriter couldn't 
     *  be initialized. In this case, the state must be "BENCHMARK_FAIL".
     */
    public void testRun4() {
        BenchmarkManager manager = new  BenchmarkManager(BENCHMARK_NAME);
        
        BenchmarkInfo info = new BenchmarkInfo();
        manager.setBenchmarkInfo(info);
        info.setName(BENCHMARK_NAME);
        info.setClassName(BENCHMARK_CLASS_NAME);
        info.setThreadNumber(THREAD_NUMBER);
        info.setRepeatNumber(REPEAT_NUMBER);
        
        LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
        manager.setLogWriter(stubLogWriter);
        DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter2();
        manager.setDataWriter(stubDataWriter);
        
        Synchronizer sync = new Synchronizer();
        manager.setSynchronizer(sync);
        
        BenchmarkContainer[] containers = new BenchmarkContainer[info.getThreadNumber()];
        for (int i = 0; i < info.getThreadNumber(); i++) {
            int containerID = i;
            containers[i] = new StubBenchmarkManagerContainer(
                containerID, info, sync, stubLogWriter, stubDataWriter);
        }
        manager.setBenchmarkContainers(containers);
        
        manager.run();
        assertEquals(BenchmarkState.BENCHMARK_FAIL, manager.getCurrentState());
    }

    /**
     *  If container's initialization failed, the finalization of "initialized
     *  container", LogWriter and DataWriter must be called.
     *  In this case, benchmark must not be executed and the state must be
     *  "BENCHMARK_FAIL".
     */
    public void testRun5() {
        BenchmarkManager manager = new  BenchmarkManager(BENCHMARK_NAME);
        
        int threadNumber = 30;
        int errorIndex = 10;
        
        BenchmarkInfo info = new BenchmarkInfo();
        manager.setBenchmarkInfo(info);
        info.setName(BENCHMARK_NAME);
        info.setClassName(BENCHMARK_CLASS_NAME);
        info.setThreadNumber(threadNumber);
        info.setRepeatNumber(REPEAT_NUMBER);
        
        LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
        manager.setLogWriter(stubLogWriter);
        DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
        manager.setDataWriter(stubDataWriter);
        
        Synchronizer sync = new Synchronizer();
        manager.setSynchronizer(sync);
        
        BenchmarkContainer[] containers = new BenchmarkContainer[info.getThreadNumber()];
        for (int i = 0; i < info.getThreadNumber(); i++) {
            int containerID = i;
            containers[i] = new StubBenchmarkManagerContainer2(
                containerID, info, sync, stubLogWriter, stubDataWriter);
        }
        manager.setBenchmarkContainers(containers);
        StubBenchmarkManagerContainer2.setErrorIndex(errorIndex);
        
        manager.run();
        
        for (int i = 0; i < threadNumber; i++) {
            if (i < errorIndex - 1) {
                assertEquals(BenchmarkState.CLEANED, containers[i].getCurrentState());
            } else if (i == errorIndex - 1) {
                assertEquals(BenchmarkState.BENCHMARK_TERMINATED, containers[i].getCurrentState());
            } else {
                assertEquals(BenchmarkState.CREATED, containers[i].getCurrentState());
            }
        }
        
        assertEquals(BenchmarkState.BENCHMARK_FAIL, manager.getCurrentState());
    }

    /**
     *  The container which got timeout must be killed.
     */
    public void testRun6() {
        BenchmarkManager manager = new  BenchmarkManager(BENCHMARK_NAME);
        BenchmarkInfo info = new BenchmarkInfo();
        manager.setBenchmarkInfo(info);
        info.setName(BENCHMARK_NAME);
        info.setClassName(BENCHMARK_CLASS_NAME2);
        info.setThreadNumber(THREAD_NUMBER);
        info.setRepeatNumber(REPEAT_NUMBER);
        
        long benchmarkTimeout = 1 * 1000;
        manager.setBenchmarkTimeout(benchmarkTimeout);
        int sleepContainerID = THREAD_NUMBER / 2;
        int sleepRepeatNumber = REPEAT_NUMBER / 2;
        long sleepTime = benchmarkTimeout * 2;
        StubBenchmarkManagerComponent2.setSleepComponentID(sleepContainerID);
        StubBenchmarkManagerComponent2.setSleepRepeatNumber(sleepRepeatNumber);
        StubBenchmarkManagerComponent2.setSleepTime(sleepTime);
        
        LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
        manager.setLogWriter(stubLogWriter);
        DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
        manager.setDataWriter(stubDataWriter);
        
        manager.run();
        
        List terminatedContainers = manager.getTerminatedContainers();
        assertEquals(1, terminatedContainers.size());
        BenchmarkContainer terminatedContainer =
            (BenchmarkContainer) terminatedContainers.get(0);
        assertEquals(sleepContainerID, terminatedContainer.getContainerID());
        assertEquals(BenchmarkState.BENCHMARK_SUCCESS, manager.getCurrentState());
    }

    /**
     *  If any containers become "BENCHMARK_TERMINATED", manager must
     *  have all other containers stopped and call finalization.
     *  In this case, the benchmark state must become "BENCHMARK_FAIL".
     */
    public void testRun7() {
        BenchmarkManager manager = new  BenchmarkManager(BENCHMARK_NAME);
        BenchmarkInfo info = new BenchmarkInfo();
        manager.setBenchmarkInfo(info);
        info.setName(BENCHMARK_NAME);
        info.setClassName(BENCHMARK_CLASS_NAME3);
        info.setThreadNumber(THREAD_NUMBER);
        info.setRepeatNumber(REPEAT_NUMBER);
        
        int terminateContainerID = THREAD_NUMBER / 2;
        int terminateRepeatNumber = REPEAT_NUMBER / 2;
        StubBenchmarkManagerComponent3.setTerminateComponentID(terminateContainerID);
        StubBenchmarkManagerComponent3.setTerminateRepeatNumber(terminateRepeatNumber);
        
        LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
        manager.setLogWriter(stubLogWriter);
        DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
        manager.setDataWriter(stubDataWriter);
        
        manager.run();
        
        List terminatedContainers = manager.getTerminatedContainers();
        assertEquals(THREAD_NUMBER, terminatedContainers.size());
        for (int i = 0; i < terminatedContainers.size(); i++) {
            BenchmarkContainer terminatedContainer =
                (BenchmarkContainer) terminatedContainers.get(i);
            assertEquals(BenchmarkState.CLEANED, terminatedContainer.getCurrentState());
        }
        assertEquals(BenchmarkState.BENCHMARK_FAIL, manager.getCurrentState());
    }

    /**
     *  The primary node test.
     *  The state must be "BENCHMARK_SUCCESS" if benchmark was succeded.
     */
    public void testRun8() {
        try {
            FileUtil.copyFiles(CONFIGURATION_FILE_NAME, TEMP_DIR);
            FileUtil.deleteFile(CONFIGURATION_FILE_NAME);
            FileUtil.copyFiles(TEST_CONFIGURATION_FILE_NAME1, DEFAULT_DIR);
            BenchmarkManager manager = new  BenchmarkManager(BENCHMARK_NAME);
            
            BenchmarkInfo info = new BenchmarkInfo();
            manager.setBenchmarkInfo(info);
            info.setName(BENCHMARK_NAME);
            info.setClassName(BENCHMARK_CLASS_NAME);
            info.setThreadNumber(THREAD_NUMBER);
            info.setRepeatNumber(REPEAT_NUMBER);
            
            LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
            manager.setLogWriter(stubLogWriter);
            DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
            manager.setDataWriter(stubDataWriter);
            
            Synchronizer sync = new Synchronizer();
            manager.setSynchronizer(sync);
            
            BenchmarkContainer[] containers = new BenchmarkContainer[info.getThreadNumber()];
            for (int i = 0; i < info.getThreadNumber(); i++) {
                int containerID = i;
                containers[i] = new StubBenchmarkManagerContainer(
                    containerID, info, sync, stubLogWriter, stubDataWriter);
            }
            manager.setBenchmarkContainers(containers);
            
            Map optionMap = new HashMap();
            optionMap.put(BenchmarkMain.MULTI_CLIENT, "primaryNode");
            manager.setOptionMap(optionMap);
            
            manager.setSocketTimeout(500);
            
            StubBenchmarkManagerSecondaryNode secondaryNode =
                new StubBenchmarkManagerSecondaryNode(5002);
            secondaryNode.start();
            StubBenchmarkManagerSecondaryNode secondaryNode2 =
                new StubBenchmarkManagerSecondaryNode(5003);
            secondaryNode2.start();
            StubBenchmarkManagerSecondaryNode secondaryNode3 =
                new StubBenchmarkManagerSecondaryNode(5004);
            secondaryNode3.start();
            
            manager.run();
            
            try {
                Thread.sleep(200);
            } catch (InterruptedException ex) {
                // ignore
            }
            
            String[] messages = secondaryNode.getMessages();
            assertEquals(2, messages.length);
            assertTrue(messages[0].startsWith("doInit"));
            assertEquals("doRun", messages[1]);
            
            String[] messages2 = secondaryNode2.getMessages();
            assertEquals(2, messages2.length);
            assertTrue(messages2[0].startsWith("doInit"));
            assertEquals("doRun", messages2[1]);
            
            String[] messages3 = secondaryNode3.getMessages();
            assertEquals(2, messages3.length);
            assertTrue(messages3[0].startsWith("doInit"));
            assertEquals("doRun", messages3[1]);
            
            assertEquals(BenchmarkState.BENCHMARK_SUCCESS, manager.getCurrentState());
        } catch (Exception ex) {
            fail(ex.getMessage());
        } finally {
            try {
                FileUtil.copyFiles(TEMP_DIR, DEFAULT_DIR);
            } catch (Exception ex) {
                fail(ex.getMessage());
            } finally {
                FileUtil.deleteFile(TEMP_DIR);
            }
        }
    }

    /**
     *  The primary node test.
     *  If the communication with secondary nodes failed before container's
     *  initialization, BenchmarkAbortException must be thrown, and
     *  the state must become "BENCHMARL_FAIL".
     */
    public void testRun9() {
        try {
            FileUtil.copyFiles(CONFIGURATION_FILE_NAME, TEMP_DIR);
            FileUtil.deleteFile(CONFIGURATION_FILE_NAME);
            FileUtil.copyFiles(TEST_CONFIGURATION_FILE_NAME1, DEFAULT_DIR);
            BenchmarkManager manager = new  BenchmarkManager(BENCHMARK_NAME);
            
            BenchmarkInfo info = new BenchmarkInfo();
            manager.setBenchmarkInfo(info);
            info.setName(BENCHMARK_NAME);
            info.setClassName(BENCHMARK_CLASS_NAME);
            info.setThreadNumber(THREAD_NUMBER);
            info.setRepeatNumber(REPEAT_NUMBER);
            
            LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
            manager.setLogWriter(stubLogWriter);
            DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
            manager.setDataWriter(stubDataWriter);
            
            Synchronizer sync = new Synchronizer();
            manager.setSynchronizer(sync);
            
            BenchmarkContainer[] containers = new BenchmarkContainer[info.getThreadNumber()];
            for (int i = 0; i < info.getThreadNumber(); i++) {
                int containerID = i;
                containers[i] = new StubBenchmarkManagerContainer(
                    containerID, info, sync, stubLogWriter, stubDataWriter);
            }
            manager.setBenchmarkContainers(containers);
            
            Map optionMap = new HashMap();
            optionMap.put(BenchmarkMain.MULTI_CLIENT, "primaryNode2");
            manager.setOptionMap(optionMap);
            
            manager.setSocketTimeout(500);
            
            StubBenchmarkManagerSecondaryNode2 secondaryNode =
                new StubBenchmarkManagerSecondaryNode2(5006);
            secondaryNode.start();
            StubBenchmarkManagerSecondaryNode2 secondaryNode3 =
                new StubBenchmarkManagerSecondaryNode2(5008);
            secondaryNode3.start();
            
            try {
                manager.run();
                fail();
            } catch (BenchmarkAbortException ex) {
                assertEquals(BenchmarkState.BENCHMARK_FAIL, manager.getCurrentState());
            }
        } catch (Exception ex) {
            fail(ex.getMessage());
        } finally {
            try {
                FileUtil.copyFiles(TEMP_DIR, DEFAULT_DIR);
            } catch (Exception ex) {
                fail(ex.getMessage());
            } finally {
                FileUtil.deleteFile(TEMP_DIR);
            }
        }
    }

    /**
     *  The primary node test.
     *  If the container initialization failed, the notification of
     *  benchmark termination must be sent to secondary nodes,
     *  "initialized containers" must be finalized, and
     *  the state must become "BENCHMARK_FAIL".
     */
    public void testRun10() {
        try {
            FileUtil.copyFiles(CONFIGURATION_FILE_NAME, TEMP_DIR);
            FileUtil.deleteFile(CONFIGURATION_FILE_NAME);
            FileUtil.copyFiles(TEST_CONFIGURATION_FILE_NAME1, DEFAULT_DIR);
            BenchmarkManager manager = new  BenchmarkManager(BENCHMARK_NAME);
            
            BenchmarkInfo info = new BenchmarkInfo();
            manager.setBenchmarkInfo(info);
            info.setName(BENCHMARK_NAME);
            info.setClassName(BENCHMARK_CLASS_NAME);
            info.setThreadNumber(THREAD_NUMBER);
            info.setRepeatNumber(REPEAT_NUMBER);
            
            LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
            manager.setLogWriter(stubLogWriter);
            DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
            manager.setDataWriter(stubDataWriter);
            
            Synchronizer sync = new Synchronizer();
            manager.setSynchronizer(sync);
            
            StubBenchmarkManagerContainer2.setErrorIndex(REPEAT_NUMBER / 2);
            BenchmarkContainer[] containers = new BenchmarkContainer[info.getThreadNumber()];
            for (int i = 0; i < info.getThreadNumber(); i++) {
                int containerID = i;
                containers[i] = new StubBenchmarkManagerContainer2(
                    containerID, info, sync, stubLogWriter, stubDataWriter);
            }
            manager.setBenchmarkContainers(containers);
            
            Map optionMap = new HashMap();
            optionMap.put(BenchmarkMain.MULTI_CLIENT, "primaryNode3");
            manager.setOptionMap(optionMap);
            
            manager.setSocketTimeout(5000);
            
            StubBenchmarkManagerSecondaryNode3 secondaryNode =
                new StubBenchmarkManagerSecondaryNode3(5010);
            secondaryNode.start();
            StubBenchmarkManagerSecondaryNode3 secondaryNode2 =
                new StubBenchmarkManagerSecondaryNode3(5011);
            secondaryNode2.start();
            StubBenchmarkManagerSecondaryNode3 secondaryNode3 =
                new StubBenchmarkManagerSecondaryNode3(5012);
            secondaryNode3.start();
            
            manager.run();
            
            try {
                Thread.sleep(200);
            } catch (InterruptedException ex) {
                // ignore
            }
            
            String[] messages = secondaryNode.getMessages();
            assertEquals(2, messages.length);
            assertTrue(messages[0].startsWith("doInit"));
            assertEquals("ERROR:init", messages[1]);
            
            String[] messages2 = secondaryNode2.getMessages();
            assertEquals(2, messages2.length);
            assertTrue(messages2[0].startsWith("doInit"));
            assertEquals("ERROR:init", messages2[1]);
            
            String[] messages3 = secondaryNode3.getMessages();
            assertEquals(2, messages3.length);
            assertTrue(messages3[0].startsWith("doInit"));
            assertEquals("ERROR:init", messages3[1]);
            
            assertEquals(BenchmarkState.BENCHMARK_FAIL, manager.getCurrentState());
        } catch (Exception ex) {
            fail(ex.getMessage());
        } finally {
            try {
                FileUtil.copyFiles(TEMP_DIR, DEFAULT_DIR);
            } catch (Exception ex) {
                fail(ex.getMessage());
            } finally {
                FileUtil.deleteFile(TEMP_DIR);
            }
        }
    }

    /**
     *  The primary node test.
     *  If the container intialization was succeed then recieved the error notification
     *  from secondary node, the manager must send the notification of benchmark
     *  termination to other secondary nodes, call the finalization of all containers,
     *  then the state must become "BENCHMARK_FAIL".
     */
    public void testRun11() {
        try {
            FileUtil.copyFiles(CONFIGURATION_FILE_NAME, TEMP_DIR);
            FileUtil.deleteFile(CONFIGURATION_FILE_NAME);
            FileUtil.copyFiles(TEST_CONFIGURATION_FILE_NAME1, DEFAULT_DIR);
            BenchmarkManager manager = new  BenchmarkManager(BENCHMARK_NAME);
            
            BenchmarkInfo info = new BenchmarkInfo();
            manager.setBenchmarkInfo(info);
            info.setName(BENCHMARK_NAME);
            info.setClassName(BENCHMARK_CLASS_NAME);
            info.setThreadNumber(THREAD_NUMBER);
            info.setRepeatNumber(REPEAT_NUMBER);
            
            LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
            manager.setLogWriter(stubLogWriter);
            DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
            manager.setDataWriter(stubDataWriter);
            
            Synchronizer sync = new Synchronizer();
            manager.setSynchronizer(sync);
            
            BenchmarkContainer[] containers = new BenchmarkContainer[info.getThreadNumber()];
            for (int i = 0; i < info.getThreadNumber(); i++) {
                int containerID = i;
                containers[i] = new StubBenchmarkManagerContainer(
                    containerID, info, sync, stubLogWriter, stubDataWriter);
            }
            manager.setBenchmarkContainers(containers);
            
            Map optionMap = new HashMap();
            optionMap.put(BenchmarkMain.MULTI_CLIENT, "primaryNode4");
            manager.setOptionMap(optionMap);
            
            manager.setSocketTimeout(5000);
            
            StubBenchmarkManagerSecondaryNode3 secondaryNode =
                new StubBenchmarkManagerSecondaryNode3(5014);
            secondaryNode.start();
            StubBenchmarkManagerSecondaryNode4 secondaryNode2 =
                new StubBenchmarkManagerSecondaryNode4(5015);
            secondaryNode2.start();
            StubBenchmarkManagerSecondaryNode3 secondaryNode3 =
                new StubBenchmarkManagerSecondaryNode3(5016);
            secondaryNode3.start();
            
            manager.run();
            
            try {
                Thread.sleep(200);
            } catch (InterruptedException ex) {
                // ignore
            }
            
            String[] messages = secondaryNode.getMessages();
            assertEquals(2, messages.length);
            assertTrue(messages[0].startsWith("doInit"));
            assertEquals("doTerminate", messages[1]);
            
            String[] messages2 = secondaryNode2.getMessages();
            assertEquals(1, messages2.length);
            assertTrue(messages[0].startsWith("doInit"));
            
            String[] messages3 = secondaryNode3.getMessages();
            assertEquals(2, messages3.length);
            assertTrue(messages3[0].startsWith("doInit"));
            assertEquals("doTerminate", messages3[1]);
            
            assertEquals(BenchmarkState.BENCHMARK_FAIL, manager.getCurrentState());
        } catch (Exception ex) {
            fail(ex.getMessage());
        } finally {
            try {
                FileUtil.copyFiles(TEMP_DIR, DEFAULT_DIR);
            } catch (Exception ex) {
                fail(ex.getMessage());
            } finally {
                FileUtil.deleteFile(TEMP_DIR);
            }
        }
    }

    /**
     *  The primary node test.
     *  If the benchmark was terminated in this node, the manager must send the
     *  notification of benchmark termination to secondary nodes, then the state
     *  must become "BENCHMARK_FAIL".
     */
    public void testRun12() {
        try {
            FileUtil.copyFiles(CONFIGURATION_FILE_NAME, TEMP_DIR);
            FileUtil.deleteFile(CONFIGURATION_FILE_NAME);
            FileUtil.copyFiles(TEST_CONFIGURATION_FILE_NAME1, DEFAULT_DIR);
            BenchmarkManager manager = new  BenchmarkManager(BENCHMARK_NAME);
            
            BenchmarkInfo info = new BenchmarkInfo();
            manager.setBenchmarkInfo(info);
            info.setName(BENCHMARK_NAME);
            info.setClassName(BENCHMARK_CLASS_NAME);
            info.setThreadNumber(THREAD_NUMBER);
            info.setRepeatNumber(REPEAT_NUMBER);
            
            LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
            manager.setLogWriter(stubLogWriter);
            DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
            manager.setDataWriter(stubDataWriter);
            
            Synchronizer sync = new Synchronizer();
            manager.setSynchronizer(sync);
            
            StubBenchmarkManagerContainer2.setErrorIndex(REPEAT_NUMBER / 2);
            
            BenchmarkContainer[] containers = new BenchmarkContainer[info.getThreadNumber()];
            for (int i = 0; i < info.getThreadNumber(); i++) {
                int containerID = i;
                containers[i] = new StubBenchmarkManagerContainer3(
                    containerID, info, sync, stubLogWriter, stubDataWriter);
            }
            manager.setBenchmarkContainers(containers);
            
            Map optionMap = new HashMap();
            optionMap.put(BenchmarkMain.MULTI_CLIENT, "primaryNode5");
            manager.setOptionMap(optionMap);
            
            manager.setSocketTimeout(500);
            
            StubBenchmarkManagerSecondaryNode5 secondaryNode =
                new StubBenchmarkManagerSecondaryNode5(5018);
            secondaryNode.start();
            StubBenchmarkManagerSecondaryNode5 secondaryNode2 =
                new StubBenchmarkManagerSecondaryNode5(5019);
            secondaryNode2.start();
            StubBenchmarkManagerSecondaryNode5 secondaryNode3 =
                new StubBenchmarkManagerSecondaryNode5(5020);
            secondaryNode3.start();
            
            manager.run();
            
            try {
                Thread.sleep(200);
            } catch (InterruptedException ex) {
                // ignore
            }
            
            String[] messages = secondaryNode.getMessages();
            assertEquals(4, messages.length);
            assertTrue(messages[0].startsWith("doInit"));
            assertEquals("doRun", messages[1]);
            assertEquals("start", messages[2]);
            assertEquals("ERROR:run", messages[3]);
            
            String[] messages2 = secondaryNode2.getMessages();
            assertEquals(4, messages2.length);
            assertTrue(messages2[0].startsWith("doInit"));
            assertEquals("doRun", messages2[1]);
            assertEquals("start", messages2[2]);
            assertEquals("ERROR:run", messages2[3]);
            
            String[] messages3 = secondaryNode3.getMessages();
            assertEquals(4, messages3.length);
            assertTrue(messages3[0].startsWith("doInit"));
            assertEquals("doRun", messages3[1]);
            assertEquals("start", messages3[2]);
            assertEquals("ERROR:run", messages3[3]);
            
            assertEquals(BenchmarkState.BENCHMARK_FAIL, manager.getCurrentState());
        } catch (Exception ex) {
            fail(ex.getMessage());
        } finally {
            try {
                FileUtil.copyFiles(TEMP_DIR, DEFAULT_DIR);
            } catch (Exception ex) {
                fail(ex.getMessage());
            } finally {
                FileUtil.deleteFile(TEMP_DIR);
            }
        }
    }

    /**
     *  The primary node test.
     *  If the manager recieved the error notification from secondary node,
     *  the manager must send the notification of benchmark termination to
     *  other secondary nodes, call containers finalziation, then become
     *  "BENCHMARK_FAIL". 
     */
    public void testRun13() {
        try {
            FileUtil.copyFiles(CONFIGURATION_FILE_NAME, TEMP_DIR);
            FileUtil.deleteFile(CONFIGURATION_FILE_NAME);
            FileUtil.copyFiles(TEST_CONFIGURATION_FILE_NAME1, DEFAULT_DIR);
            BenchmarkManager manager = new  BenchmarkManager(BENCHMARK_NAME);
            
            BenchmarkInfo info = new BenchmarkInfo();
            manager.setBenchmarkInfo(info);
            info.setName(BENCHMARK_NAME);
            info.setClassName(BENCHMARK_CLASS_NAME);
            info.setThreadNumber(THREAD_NUMBER);
            info.setRepeatNumber(REPEAT_NUMBER);
            
            LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
            manager.setLogWriter(stubLogWriter);
            DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
            manager.setDataWriter(stubDataWriter);
            
            Synchronizer sync = new Synchronizer();
            manager.setSynchronizer(sync);
            
            StubBenchmarkManagerContainer2.setErrorIndex(REPEAT_NUMBER / 2);
            
            BenchmarkContainer[] containers = new BenchmarkContainer[info.getThreadNumber()];
            for (int i = 0; i < info.getThreadNumber(); i++) {
                int containerID = i;
                containers[i] = new StubBenchmarkManagerContainer3(
                    containerID, info, sync, stubLogWriter, stubDataWriter);
            }
            manager.setBenchmarkContainers(containers);
            
            Map optionMap = new HashMap();
            optionMap.put(BenchmarkMain.MULTI_CLIENT, "primaryNode6");
            manager.setOptionMap(optionMap);
            
            manager.setSocketTimeout(500);
            
            StubBenchmarkManagerSecondaryNode5 secondaryNode =
                new StubBenchmarkManagerSecondaryNode5(5022);
            secondaryNode.start();
            StubBenchmarkManagerSecondaryNode6 secondaryNode2 =
                new StubBenchmarkManagerSecondaryNode6(5023);
            secondaryNode2.start();
            StubBenchmarkManagerSecondaryNode5 secondaryNode3 =
                new StubBenchmarkManagerSecondaryNode5(5024);
            secondaryNode3.start();
            
            manager.run();
            
            try {
                Thread.sleep(200);
            } catch (InterruptedException ex) {
                // ignore
            }
            
            String[] messages = secondaryNode.getMessages();
            assertEquals(4, messages.length);
            assertTrue(messages[0].startsWith("doInit"));
            assertEquals("doRun", messages[1]);
            assertEquals("start", messages[2]);
            assertEquals("ERROR:run", messages[3]);
            
            String[] messages2 = secondaryNode2.getMessages();
            assertEquals(3, messages2.length);
            assertTrue(messages2[0].startsWith("doInit"));
            assertEquals("doRun", messages2[1]);
            assertEquals("start", messages2[2]);
            
            String[] messages3 = secondaryNode3.getMessages();
            assertEquals(4, messages3.length);
            assertTrue(messages3[0].startsWith("doInit"));
            assertEquals("doRun", messages3[1]);
            assertEquals("start", messages3[2]);
            assertEquals("ERROR:run", messages3[3]);
            
            assertEquals(BenchmarkState.BENCHMARK_FAIL, manager.getCurrentState());
        } catch (Exception ex) {
            fail(ex.getMessage());
        } finally {
            try {
                FileUtil.copyFiles(TEMP_DIR, DEFAULT_DIR);
            } catch (Exception ex) {
                fail(ex.getMessage());
            } finally {
                FileUtil.deleteFile(TEMP_DIR);
            }
        }
    }

    /**
     *  The secondary node test.
     *  The state must be "BENCHMARK_SUCCESS" if benchmark was succeded.
     */
    public void testRun14() {
        BenchmarkManager manager = null;
        StubBenchmarkManagerMultiNode01 primary = null;
        try {
            FileUtil.copyFiles(CONFIGURATION_FILE_NAME, TEMP_DIR);
            FileUtil.deleteFile(CONFIGURATION_FILE_NAME);
            FileUtil.copyFiles(TEST_CONFIGURATION_FILE_NAME1, DEFAULT_DIR);
            manager = new  BenchmarkManager(MULTI_NODE_NAME);
            BenchmarkInfo info = new BenchmarkInfo();
            manager.setBenchmarkInfo(info);
            manager.setSocketTimeout(5000);
            info.setName(BENCHMARK_NAME);
            info.setClassName(BENCHMARK_CLASS_NAME);
            info.setThreadNumber(THREAD_NUMBER);
            info.setRepeatNumber(REPEAT_NUMBER);
            
            int newThreadNumber = 5;
            int newRepeatNumber = 5;
            
            Map optionMap = new HashMap();
            manager.setOptionMap(optionMap);
            optionMap.put(BenchmarkMain.THREAD_NUMBER, String.valueOf(newThreadNumber));
            optionMap.put(BenchmarkMain.REPEAT_NUMBER, String.valueOf(newRepeatNumber));
            optionMap.put(BenchmarkMain.MULTI_CLIENT, MULTI_NODE_NAME);
            
            LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
            manager.setLogWriter(stubLogWriter);
            DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
            manager.setDataWriter(stubDataWriter);
            
            Synchronizer sync = new Synchronizer();
            manager.setSynchronizer(sync);
            
            BenchmarkContainer[] containers =
                new BenchmarkContainer[info.getThreadNumber()];
            for (int i = 0; i < info.getThreadNumber(); i++) {
                int containerID = i;
                containers[i] = new StubBenchmarkManagerContainer(
                    containerID, info, sync, stubLogWriter, stubDataWriter);
            }
            manager.setBenchmarkContainers(containers);
            
            primary = new StubBenchmarkManagerMultiNode01();
            primary.start();
            manager.run();
            primary.join();
            
            assertEquals(BenchmarkState.BENCHMARK_SUCCESS, manager.getCurrentState());
        } catch (Exception ex) {
            fail(ex.getMessage());
        } finally {
            try {
                primary.getSocket().close();
                FileUtil.copyFiles(TEMP_DIR, DEFAULT_DIR);
            } catch (Exception ex) {
                fail(ex.getMessage());
            } finally {
                FileUtil.deleteFile(TEMP_DIR);
            }
        }
    }

    /**
     *  The secondary node test.
     *  If the communication with primary nodes failed before container's
     *  initialization, BenchmarkAbortException must be thrown, and
     *  the state must become "BENCHMARL_FAIL".
     */
    public void testRun15() {
        BenchmarkManager manager = null;
        try {
            FileUtil.copyFiles(CONFIGURATION_FILE_NAME, TEMP_DIR);
            FileUtil.deleteFile(CONFIGURATION_FILE_NAME);
            FileUtil.copyFiles(TEST_CONFIGURATION_FILE_NAME1, DEFAULT_DIR);
            manager = new  BenchmarkManager(MULTI_NODE_NAME2);
            BenchmarkInfo info = new BenchmarkInfo();
            manager.setBenchmarkInfo(info);
            manager.setSocketTimeout(5000);
            info.setName(BENCHMARK_NAME);
            info.setClassName(BENCHMARK_CLASS_NAME);
            info.setThreadNumber(THREAD_NUMBER);
            info.setRepeatNumber(REPEAT_NUMBER);
            
            int newThreadNumber = 5;
            int newRepeatNumber = 5;
            
            Map optionMap = new HashMap();
            manager.setOptionMap(optionMap);
            optionMap.put(BenchmarkMain.THREAD_NUMBER, String.valueOf(newThreadNumber));
            optionMap.put(BenchmarkMain.REPEAT_NUMBER, String.valueOf(newRepeatNumber));
            optionMap.put(BenchmarkMain.MULTI_CLIENT, MULTI_NODE_NAME2);
            
            LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
            manager.setLogWriter(stubLogWriter);
            DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
            manager.setDataWriter(stubDataWriter);
            
            Synchronizer sync = new Synchronizer();
            manager.setSynchronizer(sync);
            
            BenchmarkContainer[] containers =
                new BenchmarkContainer[info.getThreadNumber()];
            for (int i = 0; i < info.getThreadNumber(); i++) {
                int containerID = i;
                containers[i] = new StubBenchmarkManagerContainer(
                    containerID, info, sync, stubLogWriter, stubDataWriter);
            }
            manager.setBenchmarkContainers(containers);
            
            manager.run();
            fail();
        } catch (BenchmarkAbortException ex) {
            assertEquals(BenchmarkState.BENCHMARK_FAIL, manager.getCurrentState());
        } catch (Exception ex) {
            fail(ex.getMessage());
        } finally {
            try {
                FileUtil.copyFiles(TEMP_DIR, DEFAULT_DIR);
            } catch (Exception ex) {
                fail(ex.getMessage());
            } finally {
                FileUtil.deleteFile(TEMP_DIR);
            }
        }
    }

    /**
     *  The secondary node test.
     *  If the container initialization failed, the notification of
     *  benchmark termination must be sent to primary node,
     *  "initialized containers" must be finalized, and
     *  the state must become "BENCHMARK_FAIL".
     */
    public void testRun16() {
        BenchmarkManager manager = null;
        StubBenchmarkManagerMultiNode02 primary = null;
        try {
            FileUtil.copyFiles(CONFIGURATION_FILE_NAME, TEMP_DIR);
            FileUtil.deleteFile(CONFIGURATION_FILE_NAME);
            FileUtil.copyFiles(TEST_CONFIGURATION_FILE_NAME1, DEFAULT_DIR);
            manager = new  BenchmarkManager(WRONG_MULTI_NODE_NAME);
            BenchmarkInfo info = new BenchmarkInfo();
            manager.setBenchmarkInfo(info);
            manager.setSocketTimeout(5000);
            info.setName(BENCHMARK_NAME);
            info.setClassName(WRONG_BENCHMARK_CLASS_NAME);
            info.setThreadNumber(THREAD_NUMBER);
            info.setRepeatNumber(REPEAT_NUMBER);
            
            int newThreadNumber = 5;
            int newRepeatNumber = 5;
            
            Map optionMap = new HashMap();
            manager.setOptionMap(optionMap);
            optionMap.put(BenchmarkMain.THREAD_NUMBER, String.valueOf(newThreadNumber));
            optionMap.put(BenchmarkMain.REPEAT_NUMBER, String.valueOf(newRepeatNumber));
            optionMap.put(BenchmarkMain.MULTI_CLIENT, MULTI_NODE_NAME3);
            
            LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
            manager.setLogWriter(stubLogWriter);
            DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
            manager.setDataWriter(stubDataWriter);
            
            Synchronizer sync = new Synchronizer();
            manager.setSynchronizer(sync);
            
            BenchmarkContainer[] containers =
                new BenchmarkContainer[info.getThreadNumber()];
            for (int i = 0; i < info.getThreadNumber(); i++) {
                int containerID = i;
                containers[i] = new StubBenchmarkManagerContainer(
                    containerID, info, sync, stubLogWriter, stubDataWriter);
            }
            manager.setBenchmarkContainers(containers);
            
            primary = new StubBenchmarkManagerMultiNode02();
            primary.start();
            manager.run();
            primary.join();
            
            assertEquals(BenchmarkState.BENCHMARK_FAIL, manager.getCurrentState());
        } catch (BenchmarkTerminateException ex) {
            fail(ex.getMessage());
        } catch (BenchmarkAbortException ex) {
            fail(ex.getMessage());
        } catch (Exception ex) {
            fail(ex.getMessage());
        } finally {
            try {
                FileUtil.copyFiles(TEMP_DIR, DEFAULT_DIR);
                primary.getSocket().close();
            } catch (Exception ex) {
            System.out.println("test16");
                ex.printStackTrace();
                fail(ex.getMessage());
            } finally {
                FileUtil.deleteFile(TEMP_DIR);
            }
        }
    }

    /**
     *  The secondary node test.
     *  If the container intialization was succeed then recieved the error notification
     *  from primary node, the manager must call the finalization of all containers,
     *  then the state must become "BENCHMARK_FAIL".
     */
    public void testRun17() {
        BenchmarkManager manager = null;
        StubBenchmarkManagerMultiNode03 primary = null;
        try {
            FileUtil.copyFiles(CONFIGURATION_FILE_NAME, TEMP_DIR);
            FileUtil.deleteFile(CONFIGURATION_FILE_NAME);
            FileUtil.copyFiles(TEST_CONFIGURATION_FILE_NAME1, DEFAULT_DIR);
            manager = new  BenchmarkManager(MULTI_NODE_NAME4);
            BenchmarkInfo info = new BenchmarkInfo();
            manager.setBenchmarkInfo(info);
            manager.setSocketTimeout(5000);
            info.setName(BENCHMARK_NAME);
            info.setClassName(BENCHMARK_CLASS_NAME);
            info.setThreadNumber(THREAD_NUMBER);
            info.setRepeatNumber(REPEAT_NUMBER);
            
            int newThreadNumber = 5;
            int newRepeatNumber = 5;
            
            Map optionMap = new HashMap();
            manager.setOptionMap(optionMap);
            optionMap.put(BenchmarkMain.THREAD_NUMBER, String.valueOf(newThreadNumber));
            optionMap.put(BenchmarkMain.REPEAT_NUMBER, String.valueOf(newRepeatNumber));
            optionMap.put(BenchmarkMain.MULTI_CLIENT, MULTI_NODE_NAME4);
            
            LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
            manager.setLogWriter(stubLogWriter);
            DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
            manager.setDataWriter(stubDataWriter);
            
            Synchronizer sync = new Synchronizer();
            manager.setSynchronizer(sync);
            
            BenchmarkContainer[] containers =
                new BenchmarkContainer[info.getThreadNumber()];
            for (int i = 0; i < info.getThreadNumber(); i++) {
                int containerID = i;
                containers[i] = new StubBenchmarkManagerContainer(
                    containerID, info, sync, stubLogWriter, stubDataWriter);
            }
            manager.setBenchmarkContainers(containers);
            
            primary = new StubBenchmarkManagerMultiNode03();
            primary.start();
            manager.run();
            assertEquals(BenchmarkState.BENCHMARK_FAIL, manager.getCurrentState());
        } catch (Exception ex) {
            fail(ex.getMessage());
        } finally {
            try {
                primary.join();
                primary.getSocket().close();
                FileUtil.copyFiles(TEMP_DIR, DEFAULT_DIR);
            } catch (Exception ex) {
                fail(ex.getMessage());
            } finally {
                FileUtil.deleteFile(TEMP_DIR);
            }
        }
    }

    /**
     *  The secondary node test.
     *  If the benchmark was terminated in this node, the manager must send the
     *  notification of benchmark termination to primary nodes, then the state
     *  must become "BENCHMARK_FAIL".
     */
    public void testRun18() {
        BenchmarkManager manager = null;
        StubBenchmarkManagerMultiNode04 primary = null;
        try {
            FileUtil.copyFiles(CONFIGURATION_FILE_NAME, TEMP_DIR);
            FileUtil.deleteFile(CONFIGURATION_FILE_NAME);
            FileUtil.copyFiles(TEST_CONFIGURATION_FILE_NAME1, DEFAULT_DIR);
            manager = new  BenchmarkManager(MULTI_NODE_NAME5);
            BenchmarkInfo info = new BenchmarkInfo();
            manager.setBenchmarkInfo(info);
            manager.setSocketTimeout(5000);
            info.setName(BENCHMARK_NAME);
            info.setClassName(BENCHMARK_CLASS_NAME);
            info.setThreadNumber(5);
            info.setRepeatNumber(5);
            
            int newThreadNumber = 5;
            int newRepeatNumber = 5;
            
            Map optionMap = new HashMap();
            manager.setOptionMap(optionMap);
            optionMap.put(BenchmarkMain.THREAD_NUMBER, String.valueOf(newThreadNumber));
            optionMap.put(BenchmarkMain.REPEAT_NUMBER, String.valueOf(newRepeatNumber));
            optionMap.put(BenchmarkMain.MULTI_CLIENT, MULTI_NODE_NAME5);
            
            LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
            manager.setLogWriter(stubLogWriter);
            DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
            manager.setDataWriter(stubDataWriter);
            
            Synchronizer sync = new Synchronizer();
            manager.setSynchronizer(sync);
            
            BenchmarkContainer[] containers =
                new BenchmarkContainer[info.getThreadNumber()];
            StubBenchmarkManagerContainer3.setErrorIndex(3);
            for (int i = 0; i < info.getThreadNumber(); i++) {
                int containerID = i;
                containers[i] = new StubBenchmarkManagerContainer3(
                    containerID, info, sync, stubLogWriter, stubDataWriter);
            }
            manager.setBenchmarkContainers(containers);
            
            primary = new StubBenchmarkManagerMultiNode04();
            primary.start();
            manager.run();
            primary.join();
            
            assertEquals(BenchmarkState.BENCHMARK_FAIL, manager.getCurrentState());
        } catch (Exception ex) {
            fail(ex.getMessage());
        } finally {
            try {
                primary.getSocket().close();
                FileUtil.copyFiles(TEMP_DIR, DEFAULT_DIR);
            } catch (Exception ex) {
                fail(ex.getMessage());
            } finally {
                FileUtil.deleteFile(TEMP_DIR);
            }
        }
    }

    /**
     *  The secondary node test.
     *  If the manager recieved the error notification from primaray node,
     *  the manager must call containers finalziation, then become
     *  "BENCHMARK_FAIL". 
     */
    public void testRun19() {
        BenchmarkManager manager = null;
        StubBenchmarkManagerMultiNode05 primary = null;
        try {
            FileUtil.copyFiles(CONFIGURATION_FILE_NAME, TEMP_DIR);
            FileUtil.deleteFile(CONFIGURATION_FILE_NAME);
            FileUtil.copyFiles(TEST_CONFIGURATION_FILE_NAME1, DEFAULT_DIR);
            manager = new  BenchmarkManager(MULTI_NODE_NAME6);
            BenchmarkInfo info = new BenchmarkInfo();
            manager.setBenchmarkInfo(info);
            manager.setSocketTimeout(5000);
            manager.setBenchmarkTimeout(5000);
            info.setName(BENCHMARK_NAME);
            info.setClassName(BENCHMARK_CLASS_NAME);
            info.setThreadNumber(THREAD_NUMBER);
            info.setRepeatNumber(REPEAT_NUMBER);
            
            int newThreadNumber = 5;
            int newRepeatNumber = 5;
            
            Map optionMap = new HashMap();
            manager.setOptionMap(optionMap);
            optionMap.put(BenchmarkMain.THREAD_NUMBER, String.valueOf(newThreadNumber));
            optionMap.put(BenchmarkMain.REPEAT_NUMBER, String.valueOf(newRepeatNumber));
            optionMap.put(BenchmarkMain.MULTI_CLIENT, MULTI_NODE_NAME6);
            
            LogWriter stubLogWriter = new StubBenchmarkManagerLogWriter(LOG_NAME);
            manager.setLogWriter(stubLogWriter);
            DataWriter stubDataWriter = new StubBenchmarkManagerDataWriter();
            manager.setDataWriter(stubDataWriter);
            
            Synchronizer sync = new Synchronizer();
            manager.setSynchronizer(sync);
            
            BenchmarkContainer[] containers =
                new BenchmarkContainer[info.getThreadNumber()];
            for (int i = 0; i < info.getThreadNumber(); i++) {
                int containerID = i;
                containers[i] = new StubBenchmarkManagerContainer(
                    containerID, info, sync, stubLogWriter, stubDataWriter);
            }
            manager.setBenchmarkContainers(containers);
            primary = new StubBenchmarkManagerMultiNode05();
            primary.start();
            manager.start();
            primary.join();
            manager.join();
            
            // wait for changing a parameter
            Thread.sleep(1000);
            
            assertEquals(BenchmarkState.BENCHMARK_FAIL, manager.getCurrentState());
        } catch (Exception ex) {
            fail(ex.getMessage());
        } finally {
            try {
                FileUtil.copyFiles(TEMP_DIR, DEFAULT_DIR);
            } catch (Exception ex) {
                ex.printStackTrace();
                fail(ex.getMessage());
            } finally {
                FileUtil.deleteFile(TEMP_DIR);
            }
        }
    }
}
