RunnerImpl.h

Go to the documentation of this file.

////////////////////////////////////////////////////////////////////////////////
///
/// \file
/// \author Gerald Fahrnholz
/// 
/// \brief
/// Implementation of class Runner
/// 
////////////////////////////////////////////////////////////////////////////////

#pragma once

#include "TestToolBox/CommonSources/CommonDefinitions.h"
#include "TestToolBox/Runner.h"
#include "TestToolBox/Environment.h"
#include "TestToolBox/TestEvents.h"
#include "TestToolBox/Synchronizer.h"
#include "TestToolBox/ErrorEvents.h"
#include "TestToolBox/TestTime.h"
#include "TestToolBox/Protocol.h"
#include "TestToolBox/PerformanceDataSet.h"
#include <iostream>
#include <iomanip>
#include <algorithm>

///\cond do not document

#pragma warning( push )
#pragma warning( disable : 4996 ) // vsprintf declared deprecated

namespace TestToolBox
{

//************************************************************************
// class PerformanceLogger
//************************************************************************

   
namespace
{
    /// Helper class to encapsulate collecting of performance data
    class PerformanceLogger
    {
    public:

        /// Constructor
        PerformanceLogger(
            long in_checkTimeIntervalSec);

        /// Destructor
        ~PerformanceLogger();

        /// Checks if for the given test cycle collecting of performance data
        /// is necessary. If yes the data are written to stdout and to logfile
        void CheckPerformance (
            int  in_testCycle = 0,
            bool in_forceChecking = false);

    private:

        static const int COL_WIDTH = 20;

        // Utility class which encapsulates access to system performance counters
        PerformanceDataSet m_perfDataSet;

        // vector of performance counter names we are interested in
        std::vector<std::string> m_counterNames;

        // returned vector of counter values (in double format)
        std::vector<double> m_counterValues;

        // last memory consumption value, stored during last check,
        // used to calculate changes
        long m_lastValPrivateBytes;

        // Time of last checking
        long m_lastCheckTimeSec;

        // checking performsnce data every time interval
        long m_checkTimeIntervalSec;

        // number of test cycle which was recorded last
        long m_lastTestCycle;

        // Protocol file
        std::ofstream m_logFile;

    private:
        void PrepareLogging (void);
        void OpenLogFile (
            std::string const & in_perfCounterConfig);
        void CloseLogFile (void);

    }; // class PerformanceLogger


//************************************************************************
//************************************************************************


    TTB_INLINE PerformanceLogger::PerformanceLogger(
        long in_checkTimeIntervalSec)
        : m_lastValPrivateBytes  (0)
        , m_lastCheckTimeSec     (0)
        , m_checkTimeIntervalSec (in_checkTimeIntervalSec)
        , m_lastTestCycle        (0)
    {
        TRCF("PerformanceLogger");
        TRC(TL_DEBUG, "Constructor");
        PrepareLogging();
    }

//************************************************************************

    TTB_INLINE PerformanceLogger::~PerformanceLogger(void)
    {
        TRCF("~PerformanceLogger");
        TRC(TL_DEBUG, "Destructor");
        CloseLogFile();
    }

//************************************************************************

    TTB_INLINE void PerformanceLogger::PrepareLogging(void)
    {
        TRCF("PerformanceLogger::StartLogging");
        TRC(TL_DEBUG, "Executing");

        std::string computerName = TheEnvironment()->GetNameOfComputer();
        std::string appName      = TheEnvironment()->GetAppName();

        std::string perfCounterConfig = TheEnvironment()->GetCommandLineOption(
            "-perfCounterConfig");
        if (perfCounterConfig.empty())
        {
            perfCounterConfig = "XpGerman";
        }

        if (perfCounterConfig == "XpGerman")
        {
            m_counterNames.push_back("\\\\" + computerName + "\\Prozess("
                + appName + ")\\Prozessorzeit (%)");
            m_counterNames.push_back("\\\\" + computerName + "\\Prozess("
                + appName + ")\\Handleanzahl");
            m_counterNames.push_back("\\\\" + computerName + "\\Prozess("
                + appName + ")\\Threadanzahl");
            m_counterNames.push_back("\\\\" + computerName + "\\Prozess("
                + appName + ")\\Private Bytes");
        }
        else if (perfCounterConfig == "XpEnglish")
        {
            m_counterNames.push_back("\\\\" + computerName + "\\process("
                + appName + ")\\Processor Time (%)");
            m_counterNames.push_back("\\\\" + computerName + "\\process("
                + appName + ")\\Handle Count");
            m_counterNames.push_back("\\\\" + computerName + "\\process("
                + appName + ")\\Thread Count");
            m_counterNames.push_back("\\\\" + computerName + "\\process("
                + appName + ")\\Private Bytes");
        }
        else
        {
            std::cout << "Invalid config name for '-perfCounterConfig' : "
                << perfCounterConfig << std::endl << std::endl;
            abort();
        }

        std::cout << "Trying to access the specified performance counters:\n\n";

        for (std::vector<std::string>::iterator it = m_counterNames.begin();
            it!=m_counterNames.end();++it)
        {
            std::cout << *it << std::endl;
        }

        std::cout << "\nIn case of incorrect names it may need some time to detect "
            "the error...";
        bool initSucceeded = m_perfDataSet.InitPerfCounters (m_counterNames);

        if (!initSucceeded)
        {
            std::cout << "\n\nERROR: Check the name of your computer and the names of the performance counters\n"
                      << "on your system (e.g. with utility ScanPerfCounters), correct the sample code\n"
                      << "and try again! (see also the trace hints for invalid counter names) \n\n"
                      << std::endl;
            exit (1);
        }
        else // initialization succeeded, we can write the header of the output
        {
            std::cout << "\nInitialization was successful! Recording is started...\n\n";

            OpenLogFile(perfCounterConfig);

            std::ostringstream oss;
            oss
                << std::setw(COL_WIDTH) << "Test time sec"
                << std::setw(COL_WIDTH) << "Test cycle"
                << std::setw(COL_WIDTH) << "Processortime in %"
                << std::setw(COL_WIDTH) << "Handles"
                << std::setw(COL_WIDTH) << "Threads"
                << std::setw(COL_WIDTH) << "Private bytes"
                << std::setw(COL_WIDTH) << "Bytes changed by" 
                << std::setw(COL_WIDTH) << "Changes per cycle" 
                << std::endl;
            m_logFile << oss.str();
            std::cout << oss.str();
        }

    } // PerformanceLogger::PrepareLogging

//************************************************************************

    TTB_INLINE void PerformanceLogger::OpenLogFile (
        std::string const & in_perfCounterConfig)
    {
        std::string nameLogfile = TheEnvironment()->GetOutputDir() + "\\" +
          TheEnvironment()->GetAppName() + ".PerformanceData.log";
        m_logFile.open(nameLogfile.c_str());

        m_logFile <<"                    P E R F O R M A N C E   D A T A"
            << std::endl << std::endl
            << "Test application    : " << TheEnvironment()->GetExeFileFullPath()<< std::endl
            << "Source path         : " << TheEnvironment()->GetSourceDir()<< std::endl
            << "Command line args   : " << TheEnvironment()->GetArgsStr()<< std::endl
            << "Run on computer     : " << TheEnvironment()->GetNameOfComputer()<< std::endl
            << "Run by              : " << TheEnvironment()->GetNameOfUser()<< std::endl
            << "Using configuration : " << in_perfCounterConfig << " with following counters" << std::endl;

        for (std::vector<std::string>::iterator it = m_counterNames.begin();
            it!=m_counterNames.end();++it)
        {
            m_logFile << "                    : " << *it << std::endl;
        }

        m_logFile
            << Protocol::Get()->S_SEPARATOR_LINE << std::endl
            << "\nRecording started : " << TestTime::Get()->GetStartTimeDateStr()
            << std::endl << std::endl;

    } // PerformanceLogger::OpenLogFile

    //************************************************************************

    TTB_INLINE void PerformanceLogger::CloseLogFile (void)
    {
        m_logFile
            << std::endl << std::endl
            << "Recording finished: " << TestTime::Get()->GetCurTimeDateStr();
        m_logFile.close();
    
    } // PerformanceLogger::CloseLogFile

//************************************************************************

    TTB_INLINE void PerformanceLogger::CheckPerformance (
        int  in_testCycle,
        bool in_forceChecking)
    {
        long curTestTimeSec = TestTime::Get()->GetCurTimeSec();

        if (    in_forceChecking
             || (curTestTimeSec - m_lastCheckTimeSec >= m_checkTimeIntervalSec))
        {
            // Get actual counter values
            m_counterValues = m_perfDataSet.GetCurrentCounterValues();

            // Access single value within array
            long curNumPrivateBytes = static_cast<long>(m_counterValues[3]);

            double byteChanges = (m_lastValPrivateBytes == 0) ? 0
                : (curNumPrivateBytes-m_lastValPrivateBytes);
            double changesPerTestCycle = (m_lastTestCycle == in_testCycle) ? 0
                : (1.0*(curNumPrivateBytes-m_lastValPrivateBytes)/(in_testCycle-m_lastTestCycle));

            // Write formatted output
            std::ostringstream oss;
            oss
                 << std::setw(COL_WIDTH) << curTestTimeSec
                 << std::setw(COL_WIDTH) << in_testCycle
                 << std::setw(COL_WIDTH) << std::setprecision(2) << std::fixed
                     << m_counterValues[0]
                 << std::setw(COL_WIDTH) << std::setprecision(0) << m_counterValues[1]
                 << std::setw(COL_WIDTH) << std::setprecision(0) << m_counterValues[2]
                 << std::setw(COL_WIDTH) << std::setprecision(0) << m_counterValues[3]
                 << std::setw(COL_WIDTH) << std::setprecision(0) 
                     <<  byteChanges
                 << std::setw(COL_WIDTH) << std::setprecision(0) 
                     <<  changesPerTestCycle
                 << std::endl;

            m_logFile << oss.str();
            std::cout << oss.str();

            m_lastCheckTimeSec    = curTestTimeSec;
            m_lastValPrivateBytes = curNumPrivateBytes;
            m_lastTestCycle       = in_testCycle;
        }

    } // PerformanceLogger::CheckPerformance

} // anonymous namespace


//************************************************************************
// FuncInfo
//************************************************************************


TTB_INLINE FuncInfo::FuncInfo (
    std::string const & in_funcName,
    std::string const & in_fileName,
    int                 in_lineNum,
    VoidFunc            in_pFunc)
    : m_funcName (in_funcName)
    , m_fileName (in_fileName)
    , m_lineNum  (in_lineNum)
    , m_pFunc    (in_pFunc)
{
} // FuncInfo::FuncInfo


//************************************************************************
// Runner::Register
//************************************************************************


TTB_INLINE Runner::Register::Register(
        char const * in_funcName,
        char const * in_fileName,
        int          in_lineNum,
        VoidFunc     in_pFunc)
{
    TRCF("Runner::Register");
    TRC(TL_DEBUG, "Constructor in_funcName=%s", in_funcName);

    std::string fileName (in_fileName);

    std::string::size_type pos = fileName.find_last_of("\\/");
    if (pos!=std::string::npos)
    {
        fileName = fileName.substr(pos+1, fileName.length()-1);
    }

    TestToolBox::Runner::Get()->AddTestFunc(FuncInfo(
        in_funcName, fileName, in_lineNum, in_pFunc));

} // Runner::Register::Register


//************************************************************************
// Runner
//************************************************************************


TTB_INLINE Runner::Runner(void)
    : m_funcInfos ()
{
    TRCF("Runner");
    TRC(TL_DEBUG, "Constructor");

    //----- Check command line args -----

    // output mode
    OutputLevel::Enum outputLevel = OutputLevel::eREGULAR;
    TestToolBox::Environment* pEnv = TestToolBox::TheEnvironment();
    if (pEnv->IsExistingCommandLineOption("-outputLevel"))
    {
        std::string outputLevelStr = pEnv->GetCommandLineOption("-outputLevel");
        if (outputLevelStr=="SILENT")
        {
            outputLevel = OutputLevel::eSILENT;
        }
        else if (outputLevelStr=="FATAL")
        {
            outputLevel = OutputLevel::eFATAL;
        }
        else if (outputLevelStr=="ERROR")
        {
            outputLevel = OutputLevel::eERROR;
        }
        else if (outputLevelStr=="REGULAR")
        {
            outputLevel = OutputLevel::eREGULAR;
        }
        else if (outputLevelStr=="VERBOUS")
        {
            outputLevel = OutputLevel::eVERBOUS;
        }
        else
        {
            std::cout << "!!WRONG COMMANDLINE SYNTAX, invalid param for -outputMode: "
                << outputLevel << std::endl;
        }
    }

    if (    (pEnv->IsExistingCommandLineOption("-?"))
         || (pEnv->IsExistingCommandLineOption("-help")))
    {
        DisplayCommandlineSyntax();
        exit (0);
    }
    // We assume that up to now the Protocol has not yet been instantiated,
    // so we can supply params for initialization here
    Protocol::Get(
        OutputMode::M_STDOUT | OutputMode::M_OUTFILE | OutputMode::M_RPTFILE,
        outputLevel);

    TestEvents::Get()->ConnectWithProtocol (Protocol::Get());
    TestEvents::Get()->SetDetailedLog(
        true,
        pEnv->GetCommandLineOption("-prefixForReceivedTestEvents"));
    TestEvents::Get()->SetDetailedCheckLog(true);
    Synchronizer::Get()->ConnectWithEventReceiver(TestEvents::Get());

} // Runner::Runner


//************************************************************************
//************************************************************************


TTB_INLINE Runner::~Runner()
{
    TRCF("~Runner");
    TRC(TL_DEBUG, "Destructor: Begin");

    Synchronizer::Cleanup();
    TestEvents::Cleanup();
    MyErrorEvents::Cleanup();
    Protocol::Cleanup();
    TestTime::Cleanup();
    Environment::Cleanup();

    TRC(TL_DEBUG, "Destructor: End");

} // Runner::~Runner


//************************************************************************
//************************************************************************


TTB_INLINE int Runner::Execute(
    RunnerExtensionPoints* in_pExtensionPoints)
{
    TRCF("Runner-Execute");
    TRC(TL_DEBUG, "Begin");

    int    retVal              = 0;
    int    numExceptionsCaught = 0;
    double loopTimeSec       = 0.0;
    bool   loopTimeElapsed     = false;
    long   executionCounter    = 0;

    FuncInfos funcInfos = ArrangeTestSequence();
    int numberOfTestFuncs = static_cast<int>(funcInfos.size());

    PerformanceLogger* pPerfLogger = StartPerformanceLoggingIfRequired(
        loopTimeSec);

    bool repeatLoop = false;
    do
    {
        ++executionCounter;
        if (executionCounter > 1) // repeated execution
        {
            // When running a long duration test the log files are exchanged 
            // on each execution cycle, i.e. log entries are written in alternating
            // sequence to file 1/file 2. An already existing file is overridden when
            // it is reopened for the next test cycle, thus avoiding generation of
            // huge files.
            static int prefixCounter = 0;
            ++prefixCounter;
            if (prefixCounter > 2)
            {
                prefixCounter = 1;
            }
            TestToolBox::Protocol::Get()->SwitchLogfiles(ToString(prefixCounter),
                "---- Loop-Execution: File closed when starting execution "
                    + ToString(executionCounter),
                "---- Loop-Execution: File opened when starting execution "
                    + ToString(executionCounter));
        }

        // Execute single test cycle with calling all test functions
        for (FuncInfos::iterator it = funcInfos.begin();
            it != funcInfos.end(); ++it)
        {
            int numCurTestFunc = static_cast<int>(std::distance(funcInfos.begin(), it) + 1);

            std::string info = "TestFunc " + ToString(numCurTestFunc) + "/" 
                  + ToString(numberOfTestFuncs) + " " + it->m_funcName + " ("
                  + it->m_fileName + ", " + ToString(it->m_lineNum) + ")";

            CallExtensionPoints(in_pExtensionPoints,
                BEFORE_CALL_OF_TEST_FUNC, *it, info,
                numExceptionsCaught);

            CallTestFunc((*it).m_pFunc, info, numExceptionsCaught);

            CallExtensionPoints(in_pExtensionPoints,
                AFTER_CALL_OF_TEST_FUNC, *it, info,
                numExceptionsCaught);
        }

        TestEvents::Get()->CheckForUnexpectedEvents("Runner::Execute", __FILE__,__LINE__);

        if ((numExceptionsCaught > 0) || !TestEvents::Get()->AllTestsHaveSucceeded())
        {
            retVal = 1;
        }

        loopTimeElapsed = (loopTimeSec > 0)
            ? (TestToolBox::TestTime::Get()->GetCurTimeSec() >= loopTimeSec)
            : true; // regular execution without looping

        // continue with next cycle only when no error occurred
        repeatLoop = (retVal == 0 && !loopTimeElapsed);

        if (pPerfLogger)
        {
            // in any case log after last execution cycle
            pPerfLogger->CheckPerformance(executionCounter, !repeatLoop);
        }

    } while (repeatLoop);
          // break execution loop after error has been detected or configured
          // time has elapsed

    FinishPerformanceLoggingIfRequired(pPerfLogger);

    if (retVal != 0)
    {
        Protocol::Get()->Writeln(OutputLevel::eERROR, OutputMode::M_OUTFILE,
            "!!ERROR: Runner::Execute return code is " + ToString(retVal));
    }
    TRC(TL_DEBUG, "End retVal=%d", retVal);
    return retVal;

} // Runner::Execute


//************************************************************************
//************************************************************************


TTB_INLINE void Runner::DisplayCommandlineSyntax (void)
{
    std::cout << "TestRunner / TestToolBox (TTB) 2010\n\n"
              << "\nCommandline Syntax\n" << std::endl;

    std::cout << "-outputLevel SILENT | FATAL | ERROR | REGULAR | VERBOUS\n" << std::endl
              << "  adjust amount of output written to stdout\n\n"
              << "      SILENT            : no output at all\n"
              << "      FATAL             : display only fatal execution errors\n"
              << "      ERROR             : display also regular errors detected\n"
              << "                          within test cases\n"
              << "      REGULAR (default) : medium level of output, show progress\n"
              << "                          of test cases just executing\n"
              << "      VERBOUS           : maximum level of output, shows more\n"
              << "                          details about execution, also rises amount\n"
              << "                          of output written to protocol file\n";

    std::cout << "\n-selectTestFunc <expr1>,<expr2>,...\n\n"
              << "  allows filtering of the test functions to execute,\n"
              << "  you can combine several filter conditions by separating them\n"
              << "  with \",\". The filter expression will be compared with the\n"
              << "  name of the test function as defined within TTB_TEST_FUNC and\n"
              << "  TTB_TEST_FUNC_DESC (in the last case the long name is used)\n\n"
              << "  Example: Assume a test func has the name \"MySpecialTest\"\n"
              << "  Then the expressions \"My*\", \"*ialTest\", \"*eci*\" and\n"
              << "  \"MySpecialTest\" will match and \"My\", \"Test\" and \"*Z*\"\n"
              << "  will not match\n\n"
              << "  Default: no filter set, i.e. all test functions are selected\n"
              << "  for execution\n";
              
    std::cout << "\n-selectTestFile <expr1>,<expr2>,...\n\n"
              << "  allows filtering of the test functions by selecting the file,\n"
              << "  name where the test is implemented.\n"
              << "  For more details of defining filter expressions see description\n"
              << "  of option \"-selectTestFunc\"\n"
              << "  The filter conditions of both options are combined via logical\n"
              << "  AND\n";

    std::cout << "\n-sort ALPHABETICAL\n\n"
              << "  execute the test functions in alphabetical order of their names\n";

    std::cout << "\n-loopTimeSec <numSeconds>\n\n"
              << "  repeat the execution of test cases until the specified amount\n"
              << "  of time has elapsed\n"
              << "  performance data will be recorded within logfile and written to stdout\n";

    std::cout << "\n-perfDataIntervalSec <numSeconds>\n\n"
              << "  record performance data every time the given period\n"
              << "  has elapsed\n";

    std::cout << "\n-perfCounterConfig XpEnglish | XpGerman\n\n"
              << "  the name of the performance counters is set according to the\n"
              << "  windows system\n"
              << "  Default: XpGerman\n";
    
    std::cout << "\n-prefixForReceivedTestEvents <MyPrefix>\n\n"
              << "  generates modified output of test events to support direct copying\n"
              << "  to test script. E.g. setting prefix 'TTB_EXP' generates output\n"
              << "  'TTB_EXP(\"some result: 42\");' instead of '>some result: 42'\n";
    
    std::cout << "\n-checkForMemLeaks OFF | CRT_CONTROLLED | AT_MAIN_EXIT | AT_SHUTDOWN\n"
              << "                      | AT_MAIN_EXIT+AT_SHUTDOWN\n\n"
              << "    OFF            : no check for memory leaks\n"
              << "    CRT_CONTROLLED : standard procedure, C runtime checks for leaks\n"
              << "                     at prog shutdown, exit code is NOT influenced by\n"
              << "                     leak detection, use this option if you have problems\n"
              << "                     with releasing memory within main()\n"
              << "    AT_MAIN_EXIT   : check if all data on heap has been released\n"
              << "                     when main() has finished. A detected memory\n"
              << "                     leak will cause an exit code of 1\n"
              << "    AT_SHUTDOWN    : check if all data allocated since program\n"
              << "                     startup has been released at time of shutting down.\n"
              << "                     Startup and shutdown is defined by creation and\n"
              << "                     destruction of an internal static object residing within a\n"
              << "                     compiler segment of type 'lib'. A detected memory\n"
              << "                     leak will cause an exit code of 3 and a fatal system\n"
              << "                     message (which can be ignored)\n\n"
              << "  Hint: static and global data of your specific test environment may\n"
              << "  also have an effect on memory allocation and deallocation both before\n"
              << "  and after execution of main function.\n"
              << "  Default: AT_MAIN_EXIT+AT_SHUTDOWN\n";
    
    std::cout << "\n-breakAtMemAlloc <N>\n\n"
              << "  break program execution when the given memory allocation is executed.\n"
              << "  the number to specify can be derived from memory leak reports.\n"
              << "  Works only when running within debugger.\n";
    
    std::cout << "\n-? or -help" << std::endl
              << "\n  show these infos and exit test application\n";

} // Runner::DisplayCommandlineSyntax


//************************************************************************
//************************************************************************

namespace
{
    /// Function checks if the given filter expression matches with the string to compare
    /// Examples: Assume the string to compare is "MyHome"
    /// Then "My*", "*Home", "*yH* and "MyHome" will match and "My", "Home" and "*Z*"
    /// will not match
    TTB_INLINE bool FilterMatches (
        std::string const & in_filter,
        std::string const & in_stringToCompare)
    {
        std::string filter = in_filter;
        if (filter.length()==0)
        {
            return true;
        }

        bool startMustMatch = true;
        bool endMustMatch   = true;

        if (filter[0] == '*')
        {
            startMustMatch = false;
            filter.erase(0,1); // remove "*"
        }

        if (filter.length()==0)
        {
            return true;
        }

        if (filter[filter.length()-1] == '*')
        {
            endMustMatch = false;
            filter.erase(filter.length()-1,1); // remove "*"
        }

        if (filter.length()==0)
        {
            return true;
        }

        // now check if the filter string is contained within the string to compare
        size_t pos = in_stringToCompare.find(filter);
        if (pos == std::string::npos)
        {
            // string not found
            return false;
        }

        // the filter string is contained within the string to compare
        // now check if begin and end of string have to match

        if (startMustMatch && pos > 0)
        {
            // filter string is not at begin of string
            return false;
        }

        if (endMustMatch && pos !=  in_stringToCompare.length()-filter.length())
        {
            // filter string is not at end of string
            return false;
        }

        // filter condition is fulfilled
        return true;

    } // FilterMatches


    //************************************************************************


    struct FuncNameDoesNotMatchFilter
    {
        FuncNameDoesNotMatchFilter (std::vector<std::string> in_filterConditions)
            : m_filterConditions(in_filterConditions)
        {}

        bool operator() (FuncInfo const& in_funcInfo)
        {
            TRCF("Runner-FuncNameDoesNotMatchFilter");

            bool match = false;
            for (std::vector<std::string>::iterator itFilterCond = 
                 m_filterConditions.begin();
                 itFilterCond != m_filterConditions.end(); ++itFilterCond)
            {
                std::string filter = *itFilterCond;
                if (FilterMatches(filter, in_funcInfo.m_funcName))
                {
                    match = true;
                    break;
                }
            }
            bool retVal = !match;
            TRC(TL_DEBUG, "return %d for funcName %s", retVal, in_funcInfo.m_funcName.c_str());
            return retVal;
        }

        std::vector<std::string> m_filterConditions;
    };


    //************************************************************************


    struct FileNameDoesNotMatchFilter
    {
        FileNameDoesNotMatchFilter (std::vector<std::string> in_filterConditions)
            : m_filterConditions(in_filterConditions)
        {}

        bool operator() (FuncInfo const& in_funcInfo)
        {
            TRCF("Runner-FileNameDoesNotMatchFilter");

            bool match = false;
            for (std::vector<std::string>::iterator itFilterCond = 
                 m_filterConditions.begin();
                 itFilterCond != m_filterConditions.end(); ++itFilterCond)
            {
                std::string filter = *itFilterCond;
                if (FilterMatches(filter, in_funcInfo.m_fileName))
                {
                    match = true;
                    break;
                }
            }
            bool retVal = !match;
            TRC(TL_DEBUG, "return %d for funcName %s", retVal, in_funcInfo.m_funcName.c_str());
            return retVal;
        }

        std::vector<std::string> m_filterConditions;
    };


    //************************************************************************


    struct CompareAlphabetical
    {
        bool operator() (FuncInfo const& in_lhs, FuncInfo const& in_rhs )
        {
            return in_lhs.m_funcName < in_rhs.m_funcName;
        }
    };
}


//************************************************************************


TTB_INLINE FuncInfos Runner::ArrangeTestSequence (void)
{
    TRCF("Runner-ArrangeTestSequence");
    size_t numTestFuncsSpecified = m_funcInfos.size();
    TRC(TL_DEBUG, "Begin numTestFuncsSpecified=%d", m_funcInfos.size());

    // initially use the sequence without any changes
    FuncInfos funcInfos = m_funcInfos;

    TestToolBox::Environment* pEnv = TestToolBox::TheEnvironment();

    // Filter according to name of test functions
    if (pEnv->IsExistingCommandLineOption("-selectTestFunc"))
    {
        std::vector<std::string> filterConditions =
            SplitString(pEnv->GetCommandLineOption("-selectTestFunc"));
        if (filterConditions.size()>0) // there are really filter conditions set
        {
            // remove all test funcs not matching any of the filter conditions
            funcInfos.erase(std::remove_if(funcInfos.begin(), funcInfos.end(), 
                FuncNameDoesNotMatchFilter(filterConditions)), funcInfos.end());
        }
    }

    // Filter according to file name of test functions
    if (pEnv->IsExistingCommandLineOption("-selectTestFile"))
    {
        std::vector<std::string> filterConditions =
            SplitString(pEnv->GetCommandLineOption("-selectTestFile"));
        if (filterConditions.size()>0) // there are really filter conditions set
        {
            // remove all test funcs whose file name does not match with the filter conditions
            funcInfos.erase(std::remove_if(funcInfos.begin(), funcInfos.end(), 
                FileNameDoesNotMatchFilter(filterConditions)), funcInfos.end());
        }
    }

    // Check for sort options
    if (pEnv->IsExistingCommandLineOption("-sort"))
    {
        std::string optionStr = pEnv->GetCommandLineOption("-sort");
        if (optionStr=="ALPHABETICAL")
        {
            std::sort (funcInfos.begin(), funcInfos.end(), CompareAlphabetical());
        }

        // Possible extensions: generate random order
    }

    size_t numTestFuncsSelectedForExecution = funcInfos.size();
    std::ostringstream oss;
    oss
        << "\nExecuting " << numTestFuncsSelectedForExecution << " of "
        << numTestFuncsSpecified << " specified test functions\n";
    Protocol::Get()->Writeln(OutputLevel::eREGULAR, OutputMode::M_ALL, oss.str());

    TRC(TL_DEBUG, "End numTestFuncsSelectedForExecution=%d", numTestFuncsSelectedForExecution);
    return funcInfos;

} // Runner::ArrangeTestSequence


//************************************************************************
//************************************************************************


TTB_INLINE void Runner::CallTestFunc(
    VoidFunc*           in_pVoidFunc,
    std::string const & in_info,
    int &               io_numExceptionsCaught)
{
    try
    {
        Protocol::Get()->Writeln(OutputLevel::eVERBOUS,
            OutputMode::M_STDOUT | OutputMode::M_OUTFILE, in_info + ": Begin");

        in_pVoidFunc();

        Protocol::Get()->Writeln(OutputLevel::eVERBOUS,
            OutputMode::M_STDOUT | OutputMode::M_OUTFILE, in_info + ": End");
    }
    catch(std::exception& e)
    {
        ++io_numExceptionsCaught;
        {
            std::ostringstream oss;
            oss
                << "!!EXCEPTION: \"" << e.what() << "\", "
                << in_info;
             Protocol::Get()->Writeln(OutputLevel::eFATAL, OutputMode::M_ALL, oss.str());
        }
    }
    catch(...)
    {
        ++io_numExceptionsCaught;
        {
             std::ostringstream oss;
            oss << "!!UNKNOWN EXCEPTION, "
                << in_info;
             Protocol::Get()->Writeln(OutputLevel::eFATAL, OutputMode::M_ALL, oss.str());
        }
    }

} // Runner::CallTestFunc


//************************************************************************
//************************************************************************


TTB_INLINE void Runner::CallExtensionPoints (
    RunnerExtensionPoints* in_pExtensionPoints,
    CallType               in_callType,
    FuncInfo const &       in_rFuncInfo,
    std::string const &    in_info,
    int &                  io_numExceptionsCaught)
{
    if (!in_pExtensionPoints)
    {
        // no extension points defined, do nothing
        return;
    }

    std::string info = in_callType == BEFORE_CALL_OF_TEST_FUNC
        ? "RunnerExtensionPoints::BeforeCallOfTestFunction"
        : "RunnerExtensionPoints::AfterCallOfTestFunction";
    info += " " + in_info;

    try
    {   if (in_callType == BEFORE_CALL_OF_TEST_FUNC)
        {
            in_pExtensionPoints->BeforeCallOfTestFunction(in_rFuncInfo);
        }
        else
        {
            in_pExtensionPoints->AfterCallOfTestFunction(in_rFuncInfo);
        }
    }

    catch(std::exception& e)
    {
        ++io_numExceptionsCaught;
        {
            std::ostringstream oss;
            oss
                << "!!EXCEPTION: \"" << e.what() << "\", "
                << info;
             Protocol::Get()->Writeln(OutputLevel::eFATAL, OutputMode::M_ALL, oss.str());
        }
    }
    catch(...)
    {
        ++io_numExceptionsCaught;
        {
             std::ostringstream oss;
            oss << "!!UNKNOWN EXCEPTION, "
                << info;
             Protocol::Get()->Writeln(OutputLevel::eFATAL, OutputMode::M_ALL, oss.str());
        }
    }

} // Runner::CallExtensionPoints


//************************************************************************
//************************************************************************


TTB_INLINE PerformanceLogger* Runner::StartPerformanceLoggingIfRequired (
     double & out_loopTimeSec)
{
    PerformanceLogger* pPerfLogger = 0;

    TestToolBox::Environment* pEnv = TestToolBox::TheEnvironment();
    bool checkPerfData = pEnv->IsExistingCommandLineOption("-loopTimeSec");
    double loopTimeSec = pEnv->GetCommandLineOptionVal<double>("-loopTimeSec");
    long perfDataIntervalSec = 0;
    if (pEnv->IsExistingCommandLineOption("-perfDataIntervalSec"))
    {
        perfDataIntervalSec = pEnv->GetCommandLineOptionVal<long>("-perfDataIntervalSec");
    }
    else // not explicitly specified, try to choose reasonable values
    {
        if (loopTimeSec <= 15)
        {
            perfDataIntervalSec = 1;
        }
        else if (loopTimeSec <= 60)
        {
            perfDataIntervalSec = 5;
        }
        else if (loopTimeSec <= 300)
        {
            perfDataIntervalSec = 10;
        }
        else if (loopTimeSec < 1800 )
        {
            perfDataIntervalSec = 30;
        }
        else
        {
            perfDataIntervalSec = 60;
        }
    }
    out_loopTimeSec = loopTimeSec;

    if (checkPerfData)
    {
        TestToolBox::Protocol::Get()->Writeln(
            TestToolBox::OutputLevel::eREGULAR,
            TestToolBox::OutputMode::M_STDOUT, 
            "\n---- Loop-Execution: now deactivating regular output for the next " + ToString(loopTimeSec) +  " seconds...\n");
        TestToolBox::Protocol::Get()->SetOutputLevelForStdOut(TestToolBox::OutputLevel::eERROR);
        pPerfLogger = new PerformanceLogger (perfDataIntervalSec);
        pPerfLogger->CheckPerformance(0, true); // force logging before first execution cycle
    }

    return pPerfLogger;

} // Runner::StartPerformanceLoggingIfRequired


//************************************************************************
//************************************************************************


TTB_INLINE void Runner::FinishPerformanceLoggingIfRequired (
    PerformanceLogger* in_pPerformanceLogger)
{
    if (in_pPerformanceLogger)
    {
        TestToolBox::Protocol::Get()->SetOutputLevelForStdOut(
            TestToolBox::OutputLevel::eREGULAR);
        TestToolBox::Protocol::Get()->Writeln(
            TestToolBox::OutputLevel::eREGULAR,
            TestToolBox::OutputMode::M_STDOUT, 
            "\n---- Loop-Execution: ...output is activated again\n");

        delete in_pPerformanceLogger;
    }

} // Runner::FinishPerformanceLoggingIfRequired


//************************************************************************
// class AutoMemCheck
//************************************************************************


#pragma warning(disable: 4073) // Initialisierungen stehen im Initialisierungsbereich einer Bibliothek
#pragma init_seg(lib) // create before other objects
// Class to watch changes in memory consumption between calls to constructor and destructor
// The class is constructed as file global object before main() is called.
class AutoMemCheck
{
public:
    AutoMemCheck(void)
        : m_checkForMemLeaks (false)
    {
        TRCF("AutoMemCheck");
        TRC(TL_DEBUG, "Constructor");

        // Store current memory status
        _CrtMemCheckpoint(&m_memStateStartup);

        // Send all reports to STDOUT and debugger output
       _CrtSetReportMode( _CRT_WARN, _CRTDBG_MODE_FILE|_CRTDBG_MODE_DEBUG );
       _CrtSetReportFile( _CRT_WARN, _CRTDBG_FILE_STDOUT );
       _CrtSetReportMode( _CRT_ERROR, _CRTDBG_MODE_FILE|_CRTDBG_MODE_DEBUG );
       _CrtSetReportFile( _CRT_ERROR, _CRTDBG_FILE_STDOUT );
       _CrtSetReportMode( _CRT_ASSERT, _CRTDBG_MODE_FILE|_CRTDBG_MODE_DEBUG );
       _CrtSetReportFile( _CRT_ASSERT, _CRTDBG_FILE_STDOUT );

        // Define break for allocation number specified in comand line
        TestToolBox::Environment* pEnv = TestToolBox::TheEnvironment();
        int allocNum = pEnv->GetCommandLineOptionVal<int>("-breakAtMemAlloc");
        if (allocNum!=0)
        {
            TRC(TL_DEBUG, "TRYING TO BREAK AT MEMORY ALLOCATION %d", allocNum);
            _CrtSetBreakAlloc(allocNum);
        }

        // Check command line whether memory check shall be executed
        if (!pEnv->IsExistingCommandLineOption("-checkForMemLeaks"))
        {
            // as default switch on check for memory leaks
            m_checkForMemLeaks = true;
        }
        std::string checkMode = pEnv->GetCommandLineOption("-checkForMemLeaks");
        if (checkMode.find("AT_SHUTDOWN")!=checkMode.npos)
        {
            TRC(TL_DEBUG, "Checking for memleaks when shutting down was activated");
            m_checkForMemLeaks = true;
        }
        if (checkMode.find("CRT_CONTROLLED")!=checkMode.npos)
        {
            SET_CRT_DEBUG_FIELD( _CRTDBG_LEAK_CHECK_DF );
        }

        if (    (pEnv->IsExistingCommandLineOption("-?"))
             || (pEnv->IsExistingCommandLineOption("-help")))
        {
            // deactivate check if only displaying command line syntax
            m_checkForMemLeaks = false;
        }
    }

    ~AutoMemCheck( )
    {
        TRCF("~AutoMemCheck");
        TRC(TL_DEBUG, "Destructor");

        if (m_checkForMemLeaks)
        {
#ifdef _DEBUG
            // only used in debug builds
            _CrtMemState memStateShutdown,diffState;
#endif
            _CrtMemCheckpoint(&memStateShutdown);
            if (_CrtMemDifference(&diffState,&m_memStateStartup,&memStateShutdown))
            {
                TRC(TL_DEBUG, "ERROR: Detected memory leaks when shutting down (see below)");
                std::cout << "ERROR: memory leaks detected when shutting down (TTB::AutoMemCheck destructor, see below)!\n";
                // _CrtMemDumpStatistics(&diffState);
                _CrtMemDumpAllObjectsSince(&m_memStateStartup);

                // Force an exit code different from 0,
                //a fatal error text will automatically be written to stdout
                abort();
            }
        }
    }
private:

    _CrtMemState m_memStateStartup;
    bool         m_checkForMemLeaks;

};
#pragma warning(default: 4073) // restore the warning

// The static class.
static AutoMemCheck g_AutoMemCheck;


//************************************************************************
// Free functions
//************************************************************************

namespace ShortNames
{

TTB_INLINE void Act (const char* in_actualEvent, ...)
{
    UTIL_GET_FORMATTED_LINE_NO_NEWLINE (in_actualEvent);
    TestEvents::Get()->Act(formattedLine);

} // Act


//************************************************************************
//************************************************************************


TTB_INLINE void ReportStoredErrors (void)
{
    MyErrorEvents::Get()->ReportStoredErrors();

} // ReportStoredErrors

//************************************************************************

TTB_INLINE void DeleteStoredErrors (void)
{
    MyErrorEvents::Get()->DeleteStoredErrors();

} // DeleteStoredErrors

//************************************************************************

TTB_INLINE void SetIgnoreError (
    long in_errorMsgNum,
    bool in_ignore)
{
    MyErrorEvents::Get()->SetNotificationActionForErrorMsgNum(
        in_errorMsgNum, in_ignore ? ACTION_IGNORE : ACTION_NOT_SPECIFIED);

} // ReportStoredErrors

//************************************************************************

TTB_INLINE void IgnoreErrorContainingTextPattern(
    std::string const & in_text)
{
    MyErrorEvents::Get()->IgnoreErrorContainingTextPattern(in_text);

} // IgnoreErrorContainingTextPattern

//************************************************************************

TTB_INLINE void ClearStoredErrorTextPatterns (void)
{
    MyErrorEvents::Get()->ClearStoredTextPatterns();

} // ClearStoredErrorTextPatterns

//************************************************************************

TTB_INLINE void SetErrorSync (
    bool in_syncOnError)
{
    MyErrorEvents::Get()->SetErrorSync(in_syncOnError);

} // SetErrorSync

//************************************************************************

TTB_INLINE void SetImmediateErrorLog (
    bool in_immediateLog)
{
    MyErrorEvents::Get()->SetImmediateErrorLog(in_immediateLog);

} // SetImmediateErrorLog


//************************************************************************

TTB_INLINE bool ErrorOk (
    long in_extErrorId)
{
    return MyErrorEvents::Get()->ErrorOk(in_extErrorId);

} // SetImmediateErrorLog


//************************************************************************
//************************************************************************

}; // namespace ShortNames

}; // namespace TestToolBox

#pragma warning( pop )

///\endcond

---