SDL_test_harness.c File Reference
`#include "SDL_config.h"`
`#include "SDL_test.h"`
`#include <stdio.h>`
`#include <stdlib.h>`
`#include <string.h>`
`#include <time.h>`
Include dependency graph for SDL_test_harness.c:

Go to the source code of this file.

## Macros

#define | SDLTEST_INVALID_NAME_FORMAT "(Invalid)"
#define | SDLTEST_LOG_SUMMARY_FORMAT "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"
#define | SDLTEST_FINAL_RESULT_FORMAT ">>> %s '%s': %s\n"

## Functions

char * | SDLTest_GenerateRunSeed (const int length)
| Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z). More...
static Uint64 | SDLTest_GenerateExecKey (const char *runSeed, char *suiteName, char *testName, int iteration)
static SDL_TimerID | SDLTest_SetTestTimeout (int timeout, void(*callback)())
| Set timeout handler for test. More...
static SDL_NORETURN void | SDLTest_BailOut ()
| Timeout handler. Aborts test run and exits harness process. More...
static int | SDLTest_RunTest (SDLTest_TestSuiteReference *testSuite, const SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
| Execute a test using the given execution key. More...
static float | GetClock ()
int | SDLTest_RunSuites (SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
| Execute a test suite using the given run seed and execution key. More...

## Variables

static Uint32 | SDLTest_TestCaseTimeout = 3600

## Macro Definition Documentation

## SDLTEST_FINAL_RESULT_FORMAT

#define SDLTEST_FINAL_RESULT_FORMAT ">>> %s '%s': %s\n"

Definition at line 38 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites(), and SDLTest_RunTest().

## SDLTEST_INVALID_NAME_FORMAT

#define SDLTEST_INVALID_NAME_FORMAT "(Invalid)"

Definition at line 32 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites(), and SDLTest_RunTest().

## SDLTEST_LOG_SUMMARY_FORMAT

#define SDLTEST_LOG_SUMMARY_FORMAT "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"

Definition at line 35 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites().

## Function Documentation

## GetClock()

static float GetClock | ( |  | ) |

static

Definition at line 347 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites().

{
float currentClock = clock() / (float) CLOCKS_PER_SEC;
return currentClock;
}

## SDLTest_BailOut()

static SDL_NORETURN void SDLTest_BailOut | ( |  | ) |

static

Timeout handler. Aborts test run and exits harness process.

Definition at line 210 of file SDL_test_harness.c.

References SDLTest_LogError(), and TEST_ABORTED.

Referenced by SDLTest_RunTest().

{
SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
exit(TEST_ABORTED); /* bail out from the test */
}
TEST_ABORTED
#define TEST_ABORTED
Definition: SDL_test_harness.h:51
SDLTest_LogError
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103

## SDLTest_GenerateExecKey()

static Uint64 SDLTest_GenerateExecKey | ( | const char * | runSeed,
|  | char * | suiteName,
|  | char * | testName,
|  | int | iteration
| ) |  |

static

Generates an execution key for the fuzzer.

Parameters

  runSeed | The run seed to use
  suiteName | The name of the test suite
  testName | The name of the test
  iteration | The iteration count

Returns
  The generated execution key to initialize the fuzzer with.

Definition at line 101 of file SDL_test_harness.c.

References SDLTest_Md5Context::digest, NULL, SDL_ENOMEM, SDL_Error, SDL_free, SDL_malloc, SDL_memset, SDL_snprintf, SDL_strlen, SDLTest_LogError(), SDLTest_Md5Final(), SDLTest_Md5Init(), and SDLTest_Md5Update().

Referenced by SDLTest_RunSuites().

{
SDLTest_Md5Context md5Context;
Uint64 *keys;
char iterationString[16];
size_t runSeedLength;
size_t suiteNameLength;
size_t testNameLength;
size_t iterationStringLength;
size_t entireStringLength;
char *buffer;
if (runSeed == NULL || runSeed[0] == '\0') {
SDLTest_LogError("Invalid runSeed string.");
return -1;
}
if (suiteName == NULL || suiteName[0] == '\0') {
SDLTest_LogError("Invalid suiteName string.");
return -1;
}
if (testName == NULL || testName[0] == '\0') {
SDLTest_LogError("Invalid testName string.");
return -1;
}
if (iteration <= 0) {
SDLTest_LogError("Invalid iteration count.");
return -1;
}
/* Convert iteration number into a string */
SDL_memset(iterationString, 0, sizeof(iterationString));
SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
/* Combine the parameters into single string */
runSeedLength = SDL_strlen(runSeed);
suiteNameLength = SDL_strlen(suiteName);
testNameLength = SDL_strlen(testName);
iterationStringLength = SDL_strlen(iterationString);
entireStringLength = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
buffer = (char *)SDL_malloc(entireStringLength);
if (buffer == NULL) {
SDLTest_LogError("Failed to allocate buffer for execKey generation.");
SDL_Error(SDL_ENOMEM);
return 0;
}
SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
/* Hash string and use half of the digest as 64bit exec key */
SDLTest_Md5Init(&md5Context);
SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, (unsigned int) entireStringLength);
SDLTest_Md5Final(&md5Context);
SDL_free(buffer);
keys = (Uint64 *)md5Context.digest;
return keys[0];
}
SDLTest_Md5Context
Definition: SDL_test_md5.h:71
SDL_Error
#define SDL_Error
Definition: SDL_dynapi_overrides.h:115
Uint64
uint64_t Uint64
Definition: SDL_stdinc.h:194
iteration
static void iteration()
Definition: testaudiohotplug.c:84
SDL_ENOMEM
Definition: SDL_error.h:57
SDLTest_LogError
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
SDL_free
#define SDL_free
Definition: SDL_dynapi_overrides.h:377
SDLTest_Md5Update
void SDLTest_Md5Update(SDLTest_Md5Context *mdContext, unsigned char *inBuf, unsigned int inLen)
update digest from variable length data
Definition: SDL_test_md5.c:131
SDLTest_Md5Init
void SDLTest_Md5Init(SDLTest_Md5Context *mdContext)
initialize the context
Definition: SDL_test_md5.c:110
NULL
#define NULL
Definition: begin_code.h:164
buffer
GLuint buffer
Definition: SDL_opengl_glext.h:533
SDL_strlen
#define SDL_strlen
Definition: SDL_dynapi_overrides.h:393
SDLTest_Md5Final
void SDLTest_Md5Final(SDLTest_Md5Context *mdContext)
complete digest computation
Definition: SDL_test_md5.c:180
SDL_snprintf
#define SDL_snprintf
Definition: SDL_dynapi_overrides.h:40
SDL_malloc
#define SDL_malloc
Definition: SDL_dynapi_overrides.h:374
SDL_memset
#define SDL_memset
Definition: SDL_dynapi_overrides.h:386
SDLTest_Md5Context::digest
unsigned char digest[16]
Definition: SDL_test_md5.h:75

## SDLTest_GenerateRunSeed()

char* SDLTest_GenerateRunSeed | ( | const int | length | ) |

Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z).

Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z).

Note: The returned string needs to be deallocated by the caller.

Parameters

  length | The length of the seed string to generate

Returns
  The generated seed string

Definition at line 54 of file SDL_test_harness.c.

References NULL, SDL_ENOMEM, SDL_Error, SDL_malloc, SDLTest_LogError(), SDLTest_Random(), and SDLTest_RandomInitTime().

Referenced by sdltest_generateRunSeed(), and SDLTest_RunSuites().

{
char *seed = NULL;
SDLTest_RandomContext randomContext;
int counter;
/* Sanity check input */
if (length <= 0) {
SDLTest_LogError("The length of the harness seed must be >0.");
return NULL;
}
/* Allocate output buffer */
seed = (char *)SDL_malloc((length + 1) * sizeof(char));
if (seed == NULL) {
SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
SDL_Error(SDL_ENOMEM);
return NULL;
}
/* Generate a random string of alphanumeric characters */
SDLTest_RandomInitTime(&randomContext);
for (counter = 0; counter < length; counter++) {
unsigned int number = SDLTest_Random(&randomContext);
char ch = (char) (number % (91 - 48)) + 48;
if (ch >= 58 && ch <= 64) {
ch = 65;
}
seed[counter] = ch;
}
seed[length] = '\0';
return seed;
}
SDL_Error
#define SDL_Error
Definition: SDL_dynapi_overrides.h:115
SDL_ENOMEM
Definition: SDL_error.h:57
SDLTest_LogError
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
SDLTest_RandomInitTime
void SDLTest_RandomInitTime(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
Definition: SDL_test_random.c:65
SDLTest_RandomContext
Definition: SDL_test_random.h:59
SDLTest_Random
unsigned int SDLTest_Random(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
Definition: SDL_test_random.c:80
NULL
#define NULL
Definition: begin_code.h:164
counter
GLuint counter
Definition: SDL_opengl_glext.h:4992
SDL_malloc
#define SDL_malloc
Definition: SDL_dynapi_overrides.h:374
length
GLuint GLsizei GLsizei * length
Definition: SDL_opengl_glext.h:669

## SDLTest_RunSuites()

int SDLTest_RunSuites | ( | SDLTest_TestSuiteReference * | testSuites[],
|  | const char * | userRunSeed,
|  | Uint64 | userExecKey,
|  | const char * | filter,
|  | int | testIterations
| ) |  |

Execute a test suite using the given run seed and execution key.

The filter string is matched to the suite name (full comparison) to select a single suite, or if no suite matches, it is matched to the test names (full comparison) to select a single test.

Parameters

  testSuites | Suites containing the test case.
  userRunSeed | Custom run seed provided by user, or NULL to autogenerate one.
  userExecKey | Custom execution key provided by user, or 0 to autogenerate one.
  filter | Filter specification. NULL disables. Case sensitive.
  testIterations | Number of iterations to run each test case.

Returns
  Test run result; 0 when all tests passed, 1 if any tests failed.

Definition at line 367 of file SDL_test_harness.c.

References SDLTest_TestCaseReference::description, SDLTest_TestCaseReference::enabled, GetClock(), SDLTest_TestCaseReference::name, SDLTest_TestSuiteReference::name, NULL, SDL_ENOMEM, SDL_Error, SDL_FALSE, SDL_free, SDL_malloc, SDL_PRIu64, SDL_strcmp, SDL_TRUE, SDLTEST_FINAL_RESULT_FORMAT, SDLTest_GenerateExecKey(), SDLTest_GenerateRunSeed(), SDLTEST_INVALID_NAME_FORMAT, SDLTest_Log(), SDLTEST_LOG_SUMMARY_FORMAT, SDLTest_LogError(), SDLTest_RunTest(), TEST_RESULT_FAILED, TEST_RESULT_NO_ASSERT, TEST_RESULT_PASSED, TEST_RESULT_SKIPPED, and SDLTest_TestSuiteReference::testCases.

Referenced by main().

{
int totalNumberOfTests = 0;
int failedNumberOfTests = 0;
int suiteCounter;
int testCounter;
int iterationCounter;
SDLTest_TestSuiteReference *testSuite;
const SDLTest_TestCaseReference *testCase;
const char *runSeed = NULL;
char *currentSuiteName;
char *currentTestName;
Uint64 execKey;
float runStartSeconds;
float suiteStartSeconds;
float testStartSeconds;
float runEndSeconds;
float suiteEndSeconds;
float testEndSeconds;
float runtime;
int suiteFilter = 0;
char *suiteFilterName = NULL;
int testFilter = 0;
char *testFilterName = NULL;
SDL_bool forceTestRun = SDL_FALSE;
int testResult = 0;
int runResult = 0;
Uint32 totalTestFailedCount = 0;
Uint32 totalTestPassedCount = 0;
Uint32 totalTestSkippedCount = 0;
Uint32 testFailedCount = 0;
Uint32 testPassedCount = 0;
Uint32 testSkippedCount = 0;
Uint32 countSum = 0;
const SDLTest_TestCaseReference **failedTests;
/* Sanitize test iterations */
if (testIterations < 1) {
testIterations = 1;
}
/* Generate run see if we don't have one already */
if (userRunSeed == NULL || userRunSeed[0] == '\0') {
runSeed = SDLTest_GenerateRunSeed(16);
if (runSeed == NULL) {
SDLTest_LogError("Generating a random seed failed");
return 2;
}
} else {
runSeed = userRunSeed;
}
/* Reset per-run counters */
totalTestFailedCount = 0;
totalTestPassedCount = 0;
totalTestSkippedCount = 0;
/* Take time - run start */
runStartSeconds = GetClock();
/* Log run with fuzzer parameters */
SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
/* Count the total number of tests */
suiteCounter = 0;
while (testSuites[suiteCounter]) {
testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
suiteCounter++;
testCounter = 0;
while (testSuite->testCases[testCounter])
{
testCounter++;
totalNumberOfTests++;
}
}
/* Pre-allocate an array for tracking failed tests (potentially all test cases) */
failedTests = (const SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
if (failedTests == NULL) {
SDLTest_LogError("Unable to allocate cache for failed tests");
SDL_Error(SDL_ENOMEM);
return -1;
}
/* Initialize filtering */
if (filter != NULL && filter[0] != '\0') {
/* Loop over all suites to check if we have a filter match */
suiteCounter = 0;
while (testSuites[suiteCounter] && suiteFilter == 0) {
testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
suiteCounter++;
if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
/* Matched a suite name */
suiteFilter = 1;
suiteFilterName = testSuite->name;
SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
break;
}
/* Within each suite, loop over all test cases to check if we have a filter match */
testCounter = 0;
while (testSuite->testCases[testCounter] && testFilter == 0)
{
testCase = testSuite->testCases[testCounter];
testCounter++;
if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
/* Matched a test name */
suiteFilter = 1;
suiteFilterName = testSuite->name;
testFilter = 1;
testFilterName = testCase->name;
SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
break;
}
}
}
if (suiteFilter == 0 && testFilter == 0) {
SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
SDLTest_Log("Exit code: 2");
SDL_free((void *) failedTests);
return 2;
}
}
/* Loop over all suites */
suiteCounter = 0;
while(testSuites[suiteCounter]) {
testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
suiteCounter++;
/* Filter suite if flag set and we have a name */
if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
/* Skip suite */
SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
suiteCounter,
currentSuiteName);
} else {
/* Reset per-suite counters */
testFailedCount = 0;
testPassedCount = 0;
testSkippedCount = 0;
/* Take time - suite start */
suiteStartSeconds = GetClock();
/* Log suite started */
SDLTest_Log("===== Test Suite %i: '%s' started\n",
suiteCounter,
currentSuiteName);
/* Loop over all test cases */
testCounter = 0;
while(testSuite->testCases[testCounter])
{
testCase = testSuite->testCases[testCounter];
currentTestName = (char *)((testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT);
testCounter++;
/* Filter tests if flag set and we have a name */
if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
SDL_strcmp(testFilterName, testCase->name) != 0) {
/* Skip test */
SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
suiteCounter,
testCounter,
currentTestName);
} else {
/* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
if (testFilter == 1 && !testCase->enabled) {
SDLTest_Log("Force run of disabled test since test filter was set");
forceTestRun = SDL_TRUE;
}
/* Take time - test start */
testStartSeconds = GetClock();
/* Log test started */
SDLTest_Log("----- Test Case %i.%i: '%s' started",
suiteCounter,
testCounter,
currentTestName);
if (testCase->description != NULL && testCase->description[0] != '\0') {
SDLTest_Log("Test Description: '%s'",
(testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
}
/* Loop over all iterations */
iterationCounter = 0;
while(iterationCounter < testIterations)
{
iterationCounter++;
if (userExecKey != 0) {
execKey = userExecKey;
} else {
execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
}
SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey);
testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun);
if (testResult == TEST_RESULT_PASSED) {
testPassedCount++;
totalTestPassedCount++;
} else if (testResult == TEST_RESULT_SKIPPED) {
testSkippedCount++;
totalTestSkippedCount++;
} else {
testFailedCount++;
totalTestFailedCount++;
}
}
/* Take time - test end */
testEndSeconds = GetClock();
runtime = testEndSeconds - testStartSeconds;
if (runtime < 0.0f) runtime = 0.0f;
if (testIterations > 1) {
/* Log test runtime */
SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
} else {
/* Log test runtime */
SDLTest_Log("Total Test runtime: %.1f sec", runtime);
}
/* Log final test result */
switch (testResult) {
case TEST_RESULT_PASSED:
SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Passed");
break;
case TEST_RESULT_FAILED:
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Failed");
break;
case TEST_RESULT_NO_ASSERT:
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT,"Test", currentTestName, "No Asserts");
break;
}
/* Collect failed test case references for repro-step display */
if (testResult == TEST_RESULT_FAILED) {
failedTests[failedNumberOfTests] = testCase;
failedNumberOfTests++;
}
}
}
/* Take time - suite end */
suiteEndSeconds = GetClock();
runtime = suiteEndSeconds - suiteStartSeconds;
if (runtime < 0.0f) runtime = 0.0f;
/* Log suite runtime */
SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
/* Log summary and final Suite result */
countSum = testPassedCount + testFailedCount + testSkippedCount;
if (testFailedCount == 0)
{
SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Passed");
}
else
{
SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Failed");
}
}
}
/* Take time - run end */
runEndSeconds = GetClock();
runtime = runEndSeconds - runStartSeconds;
if (runtime < 0.0f) runtime = 0.0f;
/* Log total runtime */
SDLTest_Log("Total Run runtime: %.1f sec", runtime);
/* Log summary and final run result */
countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
if (totalTestFailedCount == 0)
{
runResult = 0;
SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Passed");
}
else
{
runResult = 1;
SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Failed");
}
/* Print repro steps for failed tests */
if (failedNumberOfTests > 0) {
SDLTest_Log("Harness input to repro failures:");
for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
}
}
SDL_free((void *) failedTests);
SDLTest_Log("Exit code: %d", runResult);
return runResult;
}
SDLTest_TestSuiteReference::name
char * name
Definition: SDL_test_harness.h:91
SDLTEST_INVALID_NAME_FORMAT
#define SDLTEST_INVALID_NAME_FORMAT
Definition: SDL_test_harness.c:32
SDL_TRUE
Definition: SDL_stdinc.h:142
SDLTest_TestSuiteReference
Definition: SDL_test_harness.h:89
SDLTest_GenerateRunSeed
char * SDLTest_GenerateRunSeed(const int length)
Generates a random run seed string for the harness. The generated seed will contain alphanumeric char...
Definition: SDL_test_harness.c:54
SDL_FALSE
Definition: SDL_stdinc.h:141
SDLTEST_FINAL_RESULT_FORMAT
#define SDLTEST_FINAL_RESULT_FORMAT
Definition: SDL_test_harness.c:38
f
GLfloat f
Definition: SDL_opengl_glext.h:1870
Uint32
uint32_t Uint32
Definition: SDL_stdinc.h:181
SDL_Error
#define SDL_Error
Definition: SDL_dynapi_overrides.h:115
Uint64
uint64_t Uint64
Definition: SDL_stdinc.h:194
SDLTest_TestSuiteReference::testCases
const SDLTest_TestCaseReference ** testCases
Definition: SDL_test_harness.h:95
name
GLuint const GLchar * name
Definition: SDL_opengl_glext.h:660
SDLTest_TestCaseReference::enabled
int enabled
Definition: SDL_test_harness.h:83
SDL_ENOMEM
Definition: SDL_error.h:57
TEST_RESULT_FAILED
#define TEST_RESULT_FAILED
Definition: SDL_test_harness.h:58
SDLTest_TestCaseReference::name
char * name
Definition: SDL_test_harness.h:79
SDLTest_LogError
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
SDL_free
#define SDL_free
Definition: SDL_dynapi_overrides.h:377
GetClock
static float GetClock()
Definition: SDL_test_harness.c:347
TEST_RESULT_NO_ASSERT
#define TEST_RESULT_NO_ASSERT
Definition: SDL_test_harness.h:59
NULL
#define NULL
Definition: begin_code.h:164
SDL_bool
SDL_bool
Definition: SDL_stdinc.h:139
TEST_RESULT_PASSED
#define TEST_RESULT_PASSED
Definition: SDL_test_harness.h:57
SDLTest_TestCaseReference
Definition: SDL_test_harness.h:75
SDLTEST_LOG_SUMMARY_FORMAT
#define SDLTEST_LOG_SUMMARY_FORMAT
Definition: SDL_test_harness.c:35
SDL_PRIu64
#define SDL_PRIu64
Definition: SDL_stdinc.h:216
SDLTest_Log
void SDLTest_Log(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and INFO priority.
Definition: SDL_test_log.c:85
SDLTest_GenerateExecKey
static Uint64 SDLTest_GenerateExecKey(const char *runSeed, char *suiteName, char *testName, int iteration)
Definition: SDL_test_harness.c:101
SDL_malloc
#define SDL_malloc
Definition: SDL_dynapi_overrides.h:374
SDL_strcmp
#define SDL_strcmp
Definition: SDL_dynapi_overrides.h:417
SDLTest_TestCaseReference::description
char * description
Definition: SDL_test_harness.h:81
TEST_RESULT_SKIPPED
#define TEST_RESULT_SKIPPED
Definition: SDL_test_harness.h:60
SDLTest_RunTest
static int SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, const SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
Execute a test using the given execution key.
Definition: SDL_test_harness.c:227
filter
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
Definition: SDL_opengl_glext.h:1184

## SDLTest_RunTest()

static int SDLTest_RunTest | ( | SDLTest_TestSuiteReference * | testSuite,
|  | const SDLTest_TestCaseReference * | testCase,
|  | Uint64 | execKey,
|  | SDL_bool | forceTestRun
| ) |  |

static

Execute a test using the given execution key.

Parameters

  testSuite | Suite containing the test case.
  testCase | Case to execute.
  execKey | Execution key for the fuzzer.
  forceTestRun | Force test to run even if test was disabled in suite.

Returns
  Test case result.

Definition at line 227 of file SDL_test_harness.c.

References SDLTest_TestCaseReference::description, SDLTest_TestCaseReference::enabled, SDLTest_TestCaseReference::name, SDLTest_TestSuiteReference::name, NULL, SDL_FALSE, SDL_RemoveTimer, SDLTest_AssertSummaryToTestResult(), SDLTest_BailOut(), SDLTEST_FINAL_RESULT_FORMAT, SDLTest_FuzzerInit(), SDLTest_GetFuzzerInvocationCount(), SDLTEST_INVALID_NAME_FORMAT, SDLTest_Log(), SDLTest_LogAssertSummary(), SDLTest_LogError(), SDLTest_ResetAssertSummary(), SDLTest_SetTestTimeout(), SDLTest_TestCaseTimeout, TEST_ABORTED, TEST_RESULT_FAILED, TEST_RESULT_SETUP_FAILURE, TEST_RESULT_SKIPPED, TEST_SKIPPED, TEST_STARTED, SDLTest_TestCaseReference::testCase, SDLTest_TestSuiteReference::testCases, SDLTest_TestSuiteReference::testSetUp, testSuites, and SDLTest_TestSuiteReference::testTearDown.

Referenced by SDLTest_RunSuites().

{
SDL_TimerID timer = 0;
int testCaseResult = 0;
int testResult = 0;
int fuzzerCount;
if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
{
SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
return TEST_RESULT_SETUP_FAILURE;
}
if (!testCase->enabled && forceTestRun == SDL_FALSE)
{
SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Disabled)");
return TEST_RESULT_SKIPPED;
}
/* Initialize fuzzer */
SDLTest_FuzzerInit(execKey);
/* Reset assert tracker */
SDLTest_ResetAssertSummary();
/* Set timeout timer */
timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
/* Maybe run suite initalizer function */
if (testSuite->testSetUp) {
testSuite->testSetUp(0x0);
if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite Setup", testSuite->name, "Failed");
return TEST_RESULT_SETUP_FAILURE;
}
}
/* Run test case function */
testCaseResult = testCase->testCase(0x0);
/* Convert test execution result into harness result */
if (testCaseResult == TEST_SKIPPED) {
/* Test was programatically skipped */
testResult = TEST_RESULT_SKIPPED;
} else if (testCaseResult == TEST_STARTED) {
/* Test did not return a TEST_COMPLETED value; assume it failed */
testResult = TEST_RESULT_FAILED;
} else if (testCaseResult == TEST_ABORTED) {
/* Test was aborted early; assume it failed */
testResult = TEST_RESULT_FAILED;
} else {
/* Perform failure analysis based on asserts */
testResult = SDLTest_AssertSummaryToTestResult();
}
/* Maybe run suite cleanup function (ignore failed asserts) */
if (testSuite->testTearDown) {
testSuite->testTearDown(0x0);
}
/* Cancel timeout timer */
if (timer) {
SDL_RemoveTimer(timer);
}
/* Report on asserts and fuzzer usage */
fuzzerCount = SDLTest_GetFuzzerInvocationCount();
if (fuzzerCount > 0) {
SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
}
/* Final log based on test execution result */
if (testCaseResult == TEST_SKIPPED) {
/* Test was programatically skipped */
SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Programmatically)");
} else if (testCaseResult == TEST_STARTED) {
/* Test did not return a TEST_COMPLETED value; assume it failed */
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
} else if (testCaseResult == TEST_ABORTED) {
/* Test was aborted early; assume it failed */
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (Aborted)");
} else {
SDLTest_LogAssertSummary();
}
return testResult;
}
SDLTest_TestSuiteReference::name
char * name
Definition: SDL_test_harness.h:91
TEST_ABORTED
#define TEST_ABORTED
Definition: SDL_test_harness.h:51
SDLTest_TestSuiteReference::testTearDown
SDLTest_TestCaseTearDownFp testTearDown
Definition: SDL_test_harness.h:97
SDL_FALSE
Definition: SDL_stdinc.h:141
SDLTEST_FINAL_RESULT_FORMAT
#define SDLTEST_FINAL_RESULT_FORMAT
Definition: SDL_test_harness.c:38
SDLTest_TestCaseTimeout
static Uint32 SDLTest_TestCaseTimeout
Definition: SDL_test_harness.c:41
SDLTest_TestSuiteReference::testSetUp
SDLTest_TestCaseSetUpFp testSetUp
Definition: SDL_test_harness.h:93
SDLTest_TestCaseReference::enabled
int enabled
Definition: SDL_test_harness.h:83
SDLTest_GetFuzzerInvocationCount
int SDLTest_GetFuzzerInvocationCount(void)
Definition: SDL_test_fuzzer.c:73
TEST_RESULT_FAILED
#define TEST_RESULT_FAILED
Definition: SDL_test_harness.h:58
SDLTest_TestCaseReference::name
char * name
Definition: SDL_test_harness.h:79
SDLTest_LogError
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
SDLTest_SetTestTimeout
static SDL_TimerID SDLTest_SetTestTimeout(int timeout, void(*callback)())
Set timeout handler for test.
Definition: SDL_test_harness.c:172
SDLTest_ResetAssertSummary
void SDLTest_ResetAssertSummary(void)
Resets the assert summary counters to zero.
Definition: SDL_test_assert.c:113
TEST_RESULT_SETUP_FAILURE
#define TEST_RESULT_SETUP_FAILURE
Definition: SDL_test_harness.h:61
SDLTest_LogAssertSummary
void SDLTest_LogAssertSummary(void)
Logs summary of all assertions (total, pass, fail) since last reset as INFO or ERROR.
Definition: SDL_test_assert.c:123
TEST_STARTED
#define TEST_STARTED
Definition: SDL_test_harness.h:52
SDLTest_AssertSummaryToTestResult
int SDLTest_AssertSummaryToTestResult(void)
Converts the current assert summary state to a test result.
Definition: SDL_test_assert.c:139
NULL
#define NULL
Definition: begin_code.h:164
x0
GLuint GLfloat x0
Definition: SDL_opengl_glext.h:8583
SDLTest_Log
void SDLTest_Log(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and INFO priority.
Definition: SDL_test_log.c:85
SDLTest_BailOut
static SDL_NORETURN void SDLTest_BailOut()
Timeout handler. Aborts test run and exits harness process.
Definition: SDL_test_harness.c:210
SDLTest_FuzzerInit
void SDLTest_FuzzerInit(Uint64 execKey)
Definition: SDL_test_fuzzer.c:63
TEST_RESULT_SKIPPED
#define TEST_RESULT_SKIPPED
Definition: SDL_test_harness.h:60
SDLTest_TestCaseReference::testCase
SDLTest_TestCaseFp testCase
Definition: SDL_test_harness.h:77
TEST_SKIPPED
#define TEST_SKIPPED
Definition: SDL_test_harness.h:54
SDL_RemoveTimer
#define SDL_RemoveTimer
Definition: SDL_dynapi_overrides.h:488
SDL_TimerID
int SDL_TimerID
Definition: SDL_timer.h:86

## SDLTest_SetTestTimeout()

static SDL_TimerID SDLTest_SetTestTimeout | ( | int | timeout,
|  | void(*)() | callback
| ) |  |

static

Set timeout handler for test.

Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.

Parameters

  timeout | Timeout interval in seconds.
  callback | Function that will be called after timeout has elapsed.

Returns
  Timer id or -1 on failure.

Definition at line 172 of file SDL_test_harness.c.

References callback(), NULL, SDL_AddTimer, SDL_GetError, SDL_INIT_TIMER, SDL_InitSubSystem, SDL_NORETURN, SDL_WasInit, and SDLTest_LogError().

Referenced by SDLTest_RunTest().

{
Uint32 timeoutInMilliseconds;
SDL_TimerID timerID;
if (callback == NULL) {
SDLTest_LogError("Timeout callback can't be NULL");
return -1;
}
if (timeout < 0) {
SDLTest_LogError("Timeout value must be bigger than zero.");
return -1;
}
/* Init SDL timer if not initialized before */
if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
return -1;
}
}
/* Set timer */
timeoutInMilliseconds = timeout * 1000;
timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
if (timerID == 0) {
SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
return -1;
}
return timerID;
}
SDL_GetError
#define SDL_GetError
Definition: SDL_dynapi_overrides.h:113
SDL_TimerCallback
Uint32(* SDL_TimerCallback)(Uint32 interval, void *param)
Definition: SDL_timer.h:81
SDL_InitSubSystem
#define SDL_InitSubSystem
Definition: SDL_dynapi_overrides.h:55
Uint32
uint32_t Uint32
Definition: SDL_stdinc.h:181
SDL_AddTimer
#define SDL_AddTimer
Definition: SDL_dynapi_overrides.h:487
SDLTest_LogError
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
callback
static Uint32 callback(Uint32 interval, void *param)
Definition: testtimer.c:34
NULL
#define NULL
Definition: begin_code.h:164
SDL_INIT_TIMER
#define SDL_INIT_TIMER
Definition: SDL.h:76
timeout
GLbitfield GLuint64 timeout
Definition: SDL_opengl_glext.h:1483
x0
GLuint GLfloat x0
Definition: SDL_opengl_glext.h:8583
SDL_WasInit
#define SDL_WasInit
Definition: SDL_dynapi_overrides.h:57
SDL_TimerID
int SDL_TimerID
Definition: SDL_timer.h:86

## Variable Documentation

## SDLTest_TestCaseTimeout

Uint32 SDLTest_TestCaseTimeout = 3600

static

Definition at line 41 of file SDL_test_harness.c.

Referenced by SDLTest_RunTest().