Runner.h

Go to the documentation of this file.

////////////////////////////////////////////////////////////////////////////////
///
/// \file
/// \author Gerald Fahrnholz
/// 
/// \defgroup GrpRunner Using Runner
/// @{
/// \ingroup GrpTestUtils 
/// \brief
/// Class Runner executes a user defined set of test functions
/// containing arbitrary test cases.
///
/// The test functions have to be declared within
/// macros \ref TTB_TEST_FUNC or \ref TTB_TEST_FUNC_DESC.
///
/// <b>\n Supported features</b>
///
/// - provides a controlling main function, users have only to define the
///   test functions (similar to automatic test cases within Boost::Test)
///
/// - possibility to select and sort the test functions for execution
///
/// - relevant test events and results are written to logfiles
///   - Report YourTestApp.rpt: overview of test cases and results
///   - Protocol YourTestApp.out: more detailed log file, usable for analysis of
///     test errors
///
/// - optional time controlled repeated execution of test functions
///   for long duration tests with check of performance counters
///   (e.g. consumption of memory)
///
/// - check for memory leaks
///
/// - any error detected during testing (e.g. memory leak) will result in
///   an exit code different from 0
///
///
/// <b>\n Example for a simple test application</b>
///
/// \code
/// #include "TestToolBox/RunnerImpl.h" // includes also implementation code
///
/// //include headers of your specific test object
/// #include ...
///
/// // Define name of this component for simple trace utility and set initial trace level
/// TTB_TRC_DEF_GLOBALS ("TestRunner", TL_VAL_DEBUG);
///
/// // automatic execution of main with execution of all test functions
/// // via TestToolBox::TestRunner
/// TTB_USE_DEFAULT_TEST_RUNNER();
///
/// // here follows your first test function
/// TTB_TEST_FUNC (YourFirstTestFunc)
/// {
///     // The name of the test function implicitly is the name of the first test case.
/// 
///     // Instantiate the object to test and prepare for testing
///     YourObjectUnderTest yourObject;
///     ...
///
///     // Call some interface of your object under test
///     yourObject.DoSomething();
///     ...
///
///     // Here we assume, that the reaction somehow arrives within
///     // TestToolBox::TestEvents (e.g. through synchronous or
///     // asynchronous callback; the callback object may simply generate
///     // one or more entries within TestEvents which can be immediately
///     // verified).
///
///    // Check the received result
///    TTB_EXP("CallBack-Result: 14");
///
///    // optionally proceed with further interactions with your object under test
///    ...
/// }
///
/// // here follows your second test function
/// TTB_TEST_FUNC (YourSecondTestFunc)
/// {
///     ...
/// }
/// \endcode
///
/// Hint: It may be a good advise to always use a new test function when you need a
/// new test case.
///
///
/// <b>\n Example of a test report file</b>
///
/// \code
/// Generic test protocol template / TestToolBox (TTB) 2010
/// 
/// 
///                     T E S T   P R O T O C O L
/// 
/// Date              : 30.05.2010 16:36:33
/// Test application  : c:\My Target\Debug\TestRunner.exe
/// Test protocol     : c:\My Target\Debug\TestRunner.out
/// Test report       : c:\My Target\Debug\TestRunner.rpt
/// Source path       : c:\userdata\gerald\sw\c\testtoolbox
/// Command line args : "-checkForMemLeaks" "AT_SHUTDOWN"
/// Run on computer   : SIRIUS
/// Run by            : Gerald
/// ________________________________________________________________________________
/// 
/// Executing 9 of 9 specified test functions
/// 
/// Test: YourFirstTestFunc
/// Test: YourSecondTestFunc
///      ...
/// ________________________________________________________________________________
/// 
/// Test started    : 30.05.2010 16:36:33
/// Test ended      : 30.05.2010 16:36:33
/// Duration        :            00:00:00,016
///
/// Tests performed : 9
/// Tests failed    : 0
///
/// No error detected!
/// \endcode
///
///
/// <b>\n Snippet from a test protocol file</b>
///
/// \code
/// ...
/// ================================================================================
/// Test: WaitForSingleAsyncAnswer
/// ================================================================================
///
/// Calling CalcAsync(7)
/// (Waiting for asynchronous result via callback)
///
/// >CallBack-Result: 14
/// 
/// ================================================================================
/// Test: Waiting for multiple sync answers
/// ================================================================================
/// 
///
/// <so.CalcSync          2
/// >CallBack-Result: 4
///
/// <so.CalcSync          33
/// >CallBack-Result: 66
///
/// <so.CalcSync          99
/// >CallBack-Result: 198
/// ...
/// \endcode
///
///
/// <b>\n Recommendations for organizing and running your tests</b>
///
/// - initialization and cleanup for your test environment or for single test functions
///   is possible by providing an <b>extension class</b> derived from \ref
///   TestToolBox::RunnerExtensionPoints and using macro \ref TTB_USE_TEST_RUNNER_WITH_EXTENSIONS
///
/// - you should organize your test functions in several source files, e.g.:
///   - YourTestApp.cpp: setup your test environment (e.g. define your extension
///     class, define global data used within your tests)
///   - YourTestTopicA.cpp, YourTestTopicB.cpp: implement test functions/test
///     cases and group them
///   - you can use command line option <b>-selectTestFile</b> to execute tests
///     from only one or more source files
///     
/// - you can define additional test cases within your test functions using macro
///   \ref TTB_TEST_CASE
///
/// - to improve the information written to test protocol file you can use macros
///   \ref TTB_INFO, \ref TTB_SECTION, \ref TTB_CMD, TTB_CMD1,...<br>
///   For more information see \ref TestEvents.h and \ref GrpTestEvents
///
/// - for handling asynchronous reactions you should use macros
///   \ref TTB_INIT_SYNC, \ref TTB_WAIT_SYNC, \ref TTB_SYNC, \ref TTB_SET_SYNC_TIMEOUT<br>
///   For more information see \ref TestToolBox::Synchronizer and \ref GrpSynchronizer
///
/// - for checking the observed results within your test use macros \ref TTB_EXP,
///   TTB_SEQ and TTB_EXP_VAR<br>
///   For more information see \ref TestToolBox::TestEvents and \ref TestEvents.h
///
///
/// <b>\n Command line arguments</b>
///
/// Help about possible arguments is available through param "-?":
///
/// \code
/// >MyTestApp.exe -?
///
/// TestRunner / TestToolBox (TTB) 2010
/// 
/// 
/// Commandline Syntax
/// 
/// -outputLevel SILENT | FATAL | ERROR | REGULAR | VERBOUS
/// 
///   adjust amount of output written to stdout
/// 
///       SILENT            : no output at all
///       FATAL             : display only fatal execution errors
///       ERROR             : display also regular errors detected
///                           within test cases
///       REGULAR (default) : medium level of output, show progress
///                           of test cases just executing
///       VERBOUS           : maximum level of output, shows more
///                           details about execution, also rises amount
///                           of output written to protocol file
/// 
/// -selectTestFunc <expr1>,<expr2>,...
/// 
///   allows filtering of the test functions to execute,
///   you can combine several filter conditions by separating them
///   with ",". The filter expression will be compared with the
///   name of the test function as defined within TTB_TEST_FUNC and
///   TTB_TEST_FUNC_DESC (in the last case the long name is used)
/// 
///   Example: Assume a test func has the name "MySpecialTest"
///   Then the expressions "My*", "*ialTest", "*eci*" and
///   "MySpecialTest" will match and "My", "Test" and "*Z*"
///   will not match
/// 
///   Default: no filter set, i.e. all test functions are selected
///   for execution
///
/// -selectTestFile <expr1>,<expr2>,...
/// 
///   allows filtering of the test functions by selecting the file,
///   name where the test is implemented.
///   For more details of defining filter expressions see description
///   of option "-selectTestFunc"
///   The filter conditions of both options are combined via logical
///   AND
/// 
/// -sort ALPHABETICAL
/// 
///   execute the test functions in alphabetical order of their names
/// 
/// -loopTimeSec <numSeconds>
/// 
///   repeat the execution of test cases until the specified amount
///   of time has elapsed
///   performance data will be recorded within logfile and written to stdout
/// 
/// -perfDataIntervalSec <numSeconds>
/// 
///   record performance data every time the given period
///   has elapsed
/// 
/// -perfCounterConfig XpEnglish | XpGerman
/// 
///   the name of the performance counters is set according to the
///   windows system
///   Default: XpGerman
/// 
/// -prefixForReceivedTestEvents <MyPrefix>
/// 
///   generates modified output of test events to support direct copying
///   to test script. E.g. setting prefix 'TTB_EXP' generates output
///   'TTB_EXP("some result: 42");' instead of '>some result: 42'
/// 
/// -checkForMemLeaks OFF | AT_MAIN_EXIT | AT_SHUTDOWN
///                       | AT_MAIN_EXIT+AT_SHUTDOWN
/// 
///     OFF          : no check for memory leaks
///     AT_MAIN_EXIT : check if all data on heap has been released
///                    when main() has finished. A detected memory
///                    leak will cause an exit code of 1
///     AT_SHUTDOWN  : check if all data allocated since program
///                    startup has been released at time of shutting down.
///                    Startup and shutdown is defined by creation and
///                    destruction of an internal static object residing within a
///                    compiler segment of type 'lib'. A detected memory
///                    leak will cause an exit code of 3 and a fatal system
///                    message (which can be ignored)
/// 
///   Hint: static and global data of your specific test environment may
///   also have an effect on memory allocation and deallocation both before
///   and after execution of main function.
///   Default: AT_MAIN_EXIT+AT_SHUTDOWN
/// 
/// -breakAtMemAlloc <N>
/// 
///   break program execution when the given memory allocation is executed.
///   the number to specify can be derived from memory leak reports.
///   Works only when running within debugger.
/// 
/// -? or -help
/// 
///   show these infos and exit test application
/// \endcode
///
/// \sa Runner.h (File documentation)
/// \sa TestToolBox::Runner (class documentation)
/// 
////////////////////////////////////////////////////////////////////////////////

#pragma once

#include <vector>

#include "TestToolBox/ErrorEventsImpl.h" // template code requires include of implementation
#include "TestToolBox/TestEvents.h"

#ifdef USE_TEST_INSTANCE_DLL
#    include "TestToolBox/TestInstanceDll.h"
#endif

namespace TestToolBox
{
    typedef void VoidFunc(void);
    namespace
    {
    class PerformanceLogger;
    }

    /// Info struct for a test function
    struct FuncInfo
    {
        FuncInfo (
            std::string const & in_funcName,
            std::string const & in_fileName,
            int                 in_lineNum,
            VoidFunc            in_pFunc);

        /// name of the function
        std::string    m_funcName;

        /// file where the function is defined
        std::string    m_fileName;

        /// line number where the function is defined
        int            m_lineNum;

        /// pointer to the function
        VoidFunc*      m_pFunc;
    };
    /// Vector of test functions
    typedef std::vector<FuncInfo> FuncInfos;


    //*****************************************************************************
    //*****************************************************************************


    // You have to adjust the template error class to the specific
    // types you are using in your project

    // example how to define an enum range as error category:
    // enum MyCategory {GENERAL, CONTROL, USER, COMMUNICATION};
    // typedef ErrorEvents<MyCategory,long> MyErrorEvents;
    // but often the default settings may be sufficient, i.e. error category
    // and error id are defined as long, error severity is enum DefaultSeverity
    ///\cond
    typedef ErrorEvents<> MyErrorEvents;

    __declspec(selectany) MyErrorEvents* MyErrorEvents::s_pErrorEvents = 0;
    ///\endcond


    //*****************************************************************************
    //*****************************************************************************


    /// Base class for your specific extension class
    class RunnerExtensionPoints
    {
    public:

        virtual ~RunnerExtensionPoints(void) {};

        /// Called before or after excution of a test function
        virtual void BeforeCallOfTestFunction(FuncInfo const &){}

        /// Called after excution of a test function
        virtual void AfterCallOfTestFunction(FuncInfo const &){}

    }; // class RunnerExtensionPoints


    //*****************************************************************************
    //*****************************************************************************


    /// \brief This class runs user defined test functions and organizes
    /// their execution.
    ///
    /// \nosubgrouping 
    /// \sa Runner.h (File documentation)
    /// \sa GrpRunner (Usage documentation)
    class Runner
    {
    public:

        /// Helper class to register a user defined test function
        struct Register
        {
            /// \brief Constructor adding the given test function to the list of
            /// stored test functions within singleton instance of \ref Runner
            Register(
                char const * in_funcName,
                char const * in_fileName,
                int          in_lineNum,
                VoidFunc     in_pFunc);
        };

    private:
        ///\cond

        /// Constructor
        explicit Runner(void);
        ///\endcond

    public:
        ///\cond

        /// Destructor
        virtual ~Runner();


        // Get the pointer to the only instance of this class
#ifdef USE_TEST_INSTANCE_DLL
        static Runner* Get (void)
        {return TTB_GetInstance_Runner();}
        static Runner* s_pRunner;
        static void Cleanup (void)
        {TTB_CleanupInstance_Runner();}
#else
        static Runner* Get (void)
        {if (!s_pRunner) s_pRunner = new Runner;
        return s_pRunner;}
        static Runner* s_pRunner;
        static void Cleanup (void)
        {delete s_pRunner; s_pRunner = 0;}
#endif

        ///\endcond

        /// \brief Executes all test functions according to the given command line
        /// params.
        /// You can specify your extension class (derived from \ref RunnerExtensionPoints)
        /// to add code for initialization and cleanup of test functions. This funmction
        /// is called from within macros \ref TTB_USE_DEFAULT_TEST_RUNNER and
        /// \ref TTB_USE_TEST_RUNNER_WITH_EXTENSIONS
        template<class YOUR_CLASS_TYPE>
        static int RunMain (
            char const * in_contextInfo,
            char const * in_fileName,
            long const & in_lineNum);

        /// Automatically execute the defined test functions
        int Execute(RunnerExtensionPoints* in_pExtensionPoints = 0);

    private:

        typedef enum
        {
            BEFORE_CALL_OF_TEST_FUNC,
            AFTER_CALL_OF_TEST_FUNC
        } CallType;


        /// The stored test functions 
        FuncInfos m_funcInfos;

    private:

        /// Store the given test function for later execution.
        void AddTestFunc(FuncInfo const & in_funcInfo)
        {m_funcInfos.push_back(in_funcInfo);}

        /// Display info about possible commandline args
        void DisplayCommandlineSyntax (void);

        /// \brief Select and sort the test functions according to the given
        /// command line params
        FuncInfos ArrangeTestSequence (void);

        /// Call a single test function and catch possible exceptions
        void CallTestFunc(
            VoidFunc*           in_pVoidFunc,
            std::string const & in_info,
            int &               io_numExceptionsCaught);

        /// Call an extension function and catch possible exceptions
        void CallExtensionPoints (
            RunnerExtensionPoints* in_pExtensionPoints,
            CallType               in_callType,
            FuncInfo const &       in_rFuncInfo,
            std::string const &    in_info,
            int &                  io_numExceptionsCaught);

        /// \brief Starts logging of performance data if required from command line.
        /// Returns 0 if no data collecting is needed. 
        PerformanceLogger* StartPerformanceLoggingIfRequired (
            double & out_loopTimeSec);

        /// \brief Finishes logging of performance data. 
        void FinishPerformanceLoggingIfRequired (
            PerformanceLogger* in_pPerformanceLogger);

    }; // class Runner


    //*****************************************************************************
    //*****************************************************************************

    ///\cond do not document

    template<class YOUR_CLASS_TYPE>
    int Runner::RunMain (
        char const * in_contextInfo,
        char const * in_fileName,
        long const & in_lineNum)
    {
        TRCF("Runner-RunMain")
            TRC(TL_DEBUG, "Begin");


        //-----  Check command line whether memory check shall be executed  -----

        bool checkForMemLeaks = false;
        TestToolBox::Environment* pEnv = TestToolBox::TheEnvironment();
        if (!pEnv->IsExistingCommandLineOption("-checkForMemLeaks"))
        {
            checkForMemLeaks = true;
        }
        std::string checkMode = pEnv->GetCommandLineOption("-checkForMemLeaks");
        if (checkMode.find("AT_MAIN_EXIT")!=checkMode.npos)
        {
            TRC(TL_DEBUG, "Checking for memleaks at end of main function");
            checkForMemLeaks = true;
        }


        //-----  Execute test functions and check test results  -----

        int exitCodeTest = 0;
        {
            {
                YOUR_CLASS_TYPE yourExtensionClass;

                // Execute all test functions
                exitCodeTest = TestToolBox::Runner::Get()->Execute(&yourExtensionClass);

            } // destructor of YOUR_CLASS_TYPE

            TestToolBox::TestEvents::Get()->CheckForUnexpectedEvents(
                in_contextInfo,in_fileName,in_lineNum);
            if ((exitCodeTest == 0) && (!TestToolBox::TestEvents::Get()->AllTestsHaveSucceeded()))
            {
                exitCodeTest = 1;
            }
            TestToolBox::Runner::Get()->Cleanup();
        }


        //-----  Optional check for memory leaks  -----

        int exitCodeMemCheck = 0;

        if (checkForMemLeaks)
        {
            // Remark: here it is not possible to work with memory check points
            // at begin and end of this function because before this function
            // (and main() itself) is called, there will be automatic calls to register
            // test fucntions. This means that parts of the TTB environment are
            // created before main. But by calling Runner::Cleanup() the whole environment
            // will be released. So memory check will always detect unbalanced states.
            // Therefore we simply check if all objects are relased.
            // If you could not guarantee the complete release of objects
            // deactivate this check via command line params
            exitCodeMemCheck = _CrtDumpMemoryLeaks();
            if (exitCodeMemCheck !=0)
            {
                TRC(TL_DEBUG, "ERROR: memory leaks detected (at end of Runner::RunMain, see above)!");
                std::cout << "ERROR: memory leaks detected (at end of Runner::RunMain, see above)!\n";
            }
        }


        //-----  Build final exit code  -----

        int exitCode = exitCodeTest != 0 ? exitCodeTest : exitCodeMemCheck;
        TRC(TL_DEBUG, "End returning %d", exitCode);
        return exitCode;

    } // Runner::RunMain


#ifndef USE_TEST_INSTANCE_DLL
    __declspec(selectany) Runner* Runner::s_pRunner = 0;
#endif

    ///\endcond


    //*****************************************************************************
    // Free functions for shorter notation within test script
    //*****************************************************************************

    namespace ShortNames
    {
        /// Access/create the singleton instance of Runner
        inline Runner* TestRunner (void)
        {return Runner::Get();}

        /// Short form for TestEvents::Get()->Act()
        void Act (const char* in_actualEvent, ...);

        /// Short form for MyErrorEvents::Get()->ReportStoredErrors();
        void ReportStoredErrors (void);

        /// Short form for MyErrorEvents::Get()->DeleteStoredErrors();
        void DeleteStoredErrors (void);

        /// \brief Short form for MyErrorEvents::Get()->SetNotificationActionForErrorMsgNum(
        /// msgNum, ACTION_IGNORE/ACTION_NOT_SPECIFIED);
        void SetIgnoreError (
            long in_errorMsgNum,
            bool in_ignore);

        /// \brief Short form for MyErrorEvents::Get()->IgnoreErrorContainingTextPattern()
        void IgnoreErrorContainingTextPattern(
            std::string const & in_text);

        /// \brief Short form for MyErrorEvents::Get()->ClearStoredTextPatterns()
        void ClearStoredErrorTextPatterns(void);

        /// \brief Short form for MyErrorEvents::Get()->SetErrorSync()
        void SetErrorSync (
            bool in_syncOnError);

        /// \brief Short form for MyErrorEvents::Get()->SetImmediateErrorLog()
        void SetImmediateErrorLog (
            bool in_immediateLog);

        /// \brief Short form for MyErrorEvents::Get()->ErrorOk()
        bool ErrorOk (long in_extErrorId);
    }

}; // namespace TestToolBox


//*****************************************************************************
// M A C R O S
//*****************************************************************************

/// \brief Macro to define a test function to be executed under control of
/// TestToolBox::TestRunner()
///
/// The trick for automatic registration is a simplified variant of methods
/// used within BOOST::Test (http://www.boost.org/doc/libs/1_43_0/libs/test/doc/html/index.html)
///
/// First there will be created an additional execution function which
/// checks for test errors (of preceding activities), defines a test case with
/// the name of your test function and finally calls your test function.
///
/// Second a special (file global) register object is instantiated for each
/// test function. The register object is constructed before main() will
/// be called. Within the constructor the test function is registered
/// within the TestToolBox::Runner singleton instance. When main starts
/// the Runner instance can call all registered test functions.
#define TTB_TEST_FUNC(param_name)                                   \
static void TestFunc_##param_name();                                \
static void ExecuteTestFunc_##param_name()                          \
{                                                                   \
    TestToolBox::TestEvents::Get()->CheckForUnexpectedEvents (      \
        "next test func", __FILE__,__LINE__);                       \
    TestToolBox::TestEvents::Get()->StartNewTestCase(#param_name);  \
    TestFunc_##param_name();                                        \
};                                                                  \
TestToolBox::Runner::Register regTestFunc_##param_name(#param_name, \
    __FILE__,__LINE__, &ExecuteTestFunc_##param_name);              \
static void TestFunc_##param_name()


/// \brief Same as \ref TTB_TEST_FUNC but with possibility to assign an
/// arbitrary descriptive text to the test function
///
/// The descriptive text is used to identify the test function e.g.
/// when filtering or sorting test functions via command line options
/// -selectTestFunc and -sort.
#define TTB_TEST_FUNC_DESC(param_name,param_description)                    \
static void TestFunc_##param_name();                                        \
static void ExecuteTestFunc_##param_name()                                  \
{                                                                           \
    TestToolBox::TestEvents::Get()->CheckForUnexpectedEvents (              \
        "next test func", __FILE__,__LINE__);                               \
    TestToolBox::TestEvents::Get()->StartNewTestCase(param_description);    \
    TestFunc_##param_name();                                                \
};                                                                          \
TestToolBox::Runner::Register regTestFunc_##param_name(##param_description, \
    __FILE__,__LINE__, &ExecuteTestFunc_##param_name);                      \
static void TestFunc_##param_name()


//*****************************************************************************

/// \brief Macro to define a test case.
///
/// By default TTB_TEST_FUNC and TTB_TEST_FUNC_DESC define a test function
/// which automatically defines a test case. If you need further partitioning
/// you can define additional test cases within your test function with use of
/// macro TTB_TEST_CASE.
#define TTB_TEST_CASE(DESC)                                                                      \
TestToolBox::TestEvents::Get()->CheckForUnexpectedEvents  ("next test case", __FILE__,__LINE__); \
TestToolBox::TestEvents::Get()->StartNewTestCase(DESC);


//*****************************************************************************


/// \brief Macro to define the default implementation of main function
/// executing all test cases with TestToolBox::TestRunner()
///
/// The test application will execute the following steps:
/// - TestRunner::Execute (execution of all test functions you have defined)
/// - final check for test results
/// - optional check for memory leaks
/// - main will return an exit code of 1 if an error is detected within the tests
///   or a memory leak was detected otherwise 0
#define TTB_USE_DEFAULT_TEST_RUNNER()                                       \
int _tmain(int argc, _TCHAR* argv[])                                        \
{                                                                           \
    return TestToolBox::Runner::RunMain<TestToolBox::RunnerExtensionPoints> \
        ("main",__FILE__,__LINE__);                                         \
}


//*****************************************************************************


/// \brief Defines main(argc,argv) which executes all test functions
/// via TestToolBox::TestRunner. Specific initializations can be
/// embedded in the test sequence by providing your own extension
/// class.
///
/// The test application will execute the following steps:
/// - constructor of YOUR_EXTENSION_CLASS_TYPE
/// - TestRunner::Execute (execution of all test functions you have defined)
///   - before each test function the extension method BeforeCallOfTestFunction()
///     of your extension class is called
///   - after each test function the extension method AfterCallOfTestFunction()
///     of your extension class is called
/// - destructor of YOUR_EXTENSION_CLASS_TYPE
/// - final check for test results
/// - optional check for memory leaks
/// - main will return an exit code of 1 if an error is detected within the tests
///   or a memory leak was detected otherwise 0
///
/// Hint: it is allowed to define test cases (e.g. by using TTB_TEST_CASE) also
/// within constructor and destructor of YOUR_EXTENSION_CLASS_TYPE
#define TTB_USE_TEST_RUNNER_WITH_EXTENSIONS(YOUR_EXTENSION_CLASS_TYPE) \
int _tmain(int argc, _TCHAR* argv[])                                   \
{                                                                      \
    return  TestToolBox::Runner::RunMain<YOUR_EXTENSION_CLASS_TYPE>    \
        ("main",__FILE__,__LINE__);                                    \
}                                                                                   


//*****************************************************************************
//*****************************************************************************

/// @} // endGrpRunner

---