Benchmark Tutorial

The Benchmark module allows you to write easily benchmarks framework to a project for timing critical parts and detecting slow parts of code.

In addition it automatically creates data files of these benchmarks, as well as a gnuplot file which can display the comparison curves of the benchmarks.

# Basic Usage

To create a basic benchmark, you have to follow these steps:

- Create a new benchmark
- Write the functions that wraps the functions you want to benchmark.
- Register these wrappers functions.
- Run the benchmark.
- Free the memory.

Here is a basic example of benchmark which creates two functions that will be run. These functions just print a message.

#include <stdlib.h>
#include <stdio.h>
#include <Eina.h>
static
void work1(int request)
{
printf ("work1 in progress... Request: %d\n", request);
}
static
void work2(int request)
{
printf ("work2 in progress... Request: %d\n", request);
}
int main()
{
Eina_Benchmark *test;
Eina_Array *ea;
if (!eina_init())
return EXIT_FAILURE;
test = eina_benchmark_new("test", "run");
if (!test)
goto shutdown_eina;
eina_benchmark_register(test, "work-1", EINA_BENCHMARK(work1), 200, 300, 10);
eina_benchmark_register(test, "work-2", EINA_BENCHMARK(work2), 100, 150, 5);
ea = eina_benchmark_run(test);
eina_benchmark_free(test);
eina_shutdown();
return EXIT_SUCCESS;
shutdown_eina:
eina_shutdown();
return EXIT_FAILURE;
}
Eina.h
Eina Utility library.
Eina_Array
struct _Eina_Array Eina_Array
Type for a generic one-dimensional linear data structure.
Definition eina_array.h:216
eina_benchmark_run
EINA_API Eina_Array * eina_benchmark_run(Eina_Benchmark *bench)
Runs the benchmark's registered tests.
Definition eina_benchmark.c:213
eina_benchmark_new
EINA_API Eina_Benchmark * eina_benchmark_new(const char *name, const char *run)
Creates a new array.
Definition eina_benchmark.c:138
eina_benchmark_free
EINA_API void eina_benchmark_free(Eina_Benchmark *bench)
Frees a benchmark object.
Definition eina_benchmark.c:152
Eina_Benchmark
struct _Eina_Benchmark Eina_Benchmark
Type for a benchmark.
Definition eina_benchmark.h:329
EINA_BENCHMARK
#define EINA_BENCHMARK(function)
Definition for the cast to an Eina_Benchmark_Specimens.
Definition eina_benchmark.h:345
eina_benchmark_register
EINA_API Eina_Bool eina_benchmark_register(Eina_Benchmark *bench, const char *name, Eina_Benchmark_Specimens bench_cb, int count_start, int count_end, int count_step)
Adds a test to a benchmark.
Definition eina_benchmark.c:183
eina_shutdown
EINA_API int eina_shutdown(void)
Shuts down the Eina library.
Definition eina_main.c:379
eina_init
EINA_API int eina_init(void)
Initializes the Eina library.
Definition eina_main.c:291

As "test", "run" are passed to eina_benchmark_new() and as the tests "work-1" and "work-2" are registered, the data files bench_test_run.work-1.data and bench_test_run.work-2.data will be created after the eina_benchmark_run() call. They contain four columns. The file bench_test_run.work-1.data contains for example:

# specimen experiment time starting time ending time
200 23632 2852446 2876078
210 6924 2883046 2889970
220 6467 2895962 2902429
230 6508 2908271 2914779
240 6278 2920610 2926888
250 6342 2932830 2939172
260 6252 2944954 2951206
270 6463 2956978 2963441
280 6347 2969548 2975895
290 6457 2981702 2988159

The first column (specimen) is the integer passed to the work1() function when the test is run. The second column (experiment time) is the time, in nanosecond, that work1() takes. The third and fourth columns are self-explicit.

You can see that the integer passed work1() starts from 200 and finishes at 290, with a step of 10. These values are computed withe last 3 values passed to eina_benchmark_register(). See the document of that function for the detailed behavior.

The gnuplot file will be named bench_test_run.gnuplot. Just run:

gnuplot bench_test_run.gnuplot

To create the graphic of the comparison curves. The image file is named output_test_run.png.

# More Advanced Usage

In this section, several test will be created and run. The idea is exactly the same than in the previous section, but with some basic automatic way to run all the benchmarks. The following code benchmarks some Eina converts functions, and some Eina containers types:

#include <stdlib.h>
#include <stdio.h>
#include <time.h>
#include <Eina.h>
static void bench_convert(Eina_Benchmark *bench);
static void bench_container(Eina_Benchmark *bench);
typedef struct _Benchmark_Case Benchmark_Case;
struct _Benchmark_Case
{
const char *bench_case;
void (*build)(Eina_Benchmark *bench);
};
static const Benchmark_Case benchmarks[] = {
{ "Bench 1", bench_convert },
{ "Bench 2", bench_container },
{ NULL, NULL }
};
static
void convert1(int request)
{
char tmp[128];
int i;
for (i = 0; i < request; ++i)
eina_convert_itoa(rand(), tmp);
}
static
void convert2(int request)
{
char tmp[128];
int i;
for (i = 0; i < request; ++i)
eina_convert_xtoa(rand(), tmp);
}
static void
bench_convert(Eina_Benchmark *bench)
{
eina_benchmark_register(bench, "convert-1", EINA_BENCHMARK(convert1), 200, 400, 10);
eina_benchmark_register(bench, "convert-2", EINA_BENCHMARK(convert2), 200, 400, 10);
}
static
void array(int request)
{
Eina_Array *array;
Eina_Array_Iterator it;
int *data;
int i;
array = eina_array_new(64);
for (i = 0; i < request; ++i)
{
data = (int *)malloc(sizeof(int));
if (!data) continue;
*data = rand();
eina_array_push(array, data);
}
EINA_ARRAY_ITER_NEXT(array, i, data, it)
free(data);
eina_array_free(array);
}
static
void list(int request)
{
Eina_List *l = NULL;
int *data;
int i;
for (i = 0; i < request; ++i)
{
data = (int *)malloc(sizeof(int));
if (!data) continue;
*data = rand();
l = eina_list_prepend(l, data);
}
while (l)
{
free(eina_list_data_get(l));
l = eina_list_remove_list(l, l);
}
}
static void
bench_container(Eina_Benchmark *bench)
{
eina_benchmark_register(bench, "array", EINA_BENCHMARK(array), 200, 300, 10);
eina_benchmark_register(bench, "list", EINA_BENCHMARK(list), 200, 300, 10);
}
int main()
{
Eina_Benchmark *test;
Eina_Array *ea;
unsigned int i;
if (!eina_init())
return EXIT_FAILURE;
for (i = 0; benchmarks[i].bench_case != NULL; ++i)
{
test = eina_benchmark_new(benchmarks[i].bench_case, "Benchmark example");
if (!test)
continue;
benchmarks[i].build(test);
ea = eina_benchmark_run(test);
eina_benchmark_free(test);
}
eina_shutdown();
return EXIT_SUCCESS;
}
Eina_Array_Iterator
void ** Eina_Array_Iterator
Type for an iterator on arrays, used with EINA_ARRAY_ITER_NEXT.
Definition eina_array.h:222
eina_array_free
EINA_API void eina_array_free(Eina_Array *array)
Frees an array.
Definition eina_array.c:295
EINA_ARRAY_ITER_NEXT
#define EINA_ARRAY_ITER_NEXT(array, index, item, iterator)
Iterates through an array's elements.
Definition eina_array.h:507
eina_array_push
static Eina_Bool eina_array_push(Eina_Array *array, const void *data)
Appends a data item to an array.
eina_array_new
EINA_API Eina_Array * eina_array_new(unsigned int step)
Creates a new array.
Definition eina_array.c:276
eina_convert_itoa
EINA_API int eina_convert_itoa(int n, char *s)
Converts an integer number to a string in decimal base.
Definition eina_convert.c:154
eina_convert_xtoa
EINA_API int eina_convert_xtoa(unsigned int n, char *s)
Converts an integer number to a string in hexadecimal base.
Definition eina_convert.c:179
eina_list_prepend
EINA_API Eina_List * eina_list_prepend(Eina_List *list, const void *data)
Prepends the given data to the given linked list.
Definition eina_list.c:618
Eina_List
struct _Eina_List Eina_List
Type for a generic double linked list.
Definition eina_list.h:304
eina_list_remove_list
EINA_API Eina_List * eina_list_remove_list(Eina_List *list, Eina_List *remove_list)
Removes the specified list node.
Definition eina_list.c:786

gnuplot can be used to see how are performed the convert functions together, as well as how are performed the containers. So it is now easy to see that the hexadecimal convert function is faster than the decimal one, and that arrays are faster than lists.

You can improve all that by executing automatically gnuplot in your program, or integrate the Eina benchmark framework in an autotooled project. See that page for more information.