Loading...
Searching...
No Matches
Tutorial - Part 5: Capturing Video Frames

Tutorial - Part 4: Playing A Tone | Index | Tutorial - Part 6: Binding Objects

In this tutorial we show how to use a stream to capture a stream of video frames.

Even though we are now working with a different media type and we are capturing instead of playback, you will see that this example is very similar to Tutorial - Part 4: Playing A Tone.

Let's take a look at the code before we break it down:

#include <spa/param/video/format-utils.h>
#include <spa/debug/types.h>
#include <spa/param/video/type-info.h>
#include <pipewire/pipewire.h>
struct data {
struct pw_main_loop *loop;
struct pw_stream *stream;
struct spa_video_info format;
};
/* [on_process] */
static void on_process(void *userdata)
{
struct data *data = userdata;
struct pw_buffer *b;
struct spa_buffer *buf;
if ((b = pw_stream_dequeue_buffer(data->stream)) == NULL) {
pw_log_warn("out of buffers: %m");
return;
}
buf = b->buffer;
if (buf->datas[0].data == NULL)
return;
printf("got a frame of size %d\n", buf->datas[0].chunk->size);
pw_stream_queue_buffer(data->stream, b);
}
/* [on_process] */
static void on_param_changed(void *userdata, uint32_t id, const struct spa_pod *param)
{
struct data *data = userdata;
if (param == NULL || id != SPA_PARAM_Format)
return;
if (spa_format_parse(param,
&data->format.media_type,
&data->format.media_subtype) < 0)
return;
if (data->format.media_type != SPA_MEDIA_TYPE_video ||
data->format.media_subtype != SPA_MEDIA_SUBTYPE_raw)
return;
if (spa_format_video_raw_parse(param, &data->format.info.raw) < 0)
return;
printf("got video format:\n");
printf(" format: %d (%s)\n", data->format.info.raw.format,
spa_debug_type_find_name(spa_type_video_format,
data->format.info.raw.format));
printf(" size: %dx%d\n", data->format.info.raw.size.width,
data->format.info.raw.size.height);
printf(" framerate: %d/%d\n", data->format.info.raw.framerate.num,
data->format.info.raw.framerate.denom);
}
static const struct pw_stream_events stream_events = {
PW_VERSION_STREAM_EVENTS,
.param_changed = on_param_changed,
.process = on_process,
};
int main(int argc, char *argv[])
{
struct data data = { 0, };
const struct spa_pod *params[1];
uint8_t buffer[1024];
struct spa_pod_builder b = SPA_POD_BUILDER_INIT(buffer, sizeof(buffer));
struct pw_properties *props;
pw_init(&argc, &argv);
data.loop = pw_main_loop_new(NULL);
props = pw_properties_new(PW_KEY_MEDIA_TYPE, "Video",
PW_KEY_MEDIA_CATEGORY, "Capture",
PW_KEY_MEDIA_ROLE, "Camera",
NULL);
if (argc > 1)
pw_properties_set(props, PW_KEY_TARGET_OBJECT, argv[1]);
data.stream = pw_stream_new_simple(
pw_main_loop_get_loop(data.loop),
"video-capture",
props,
&stream_events,
&data);
params[0] = spa_pod_builder_add_object(&b,
SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat,
SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video),
SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw),
SPA_FORMAT_VIDEO_format, SPA_POD_CHOICE_ENUM_Id(7,
SPA_VIDEO_FORMAT_RGB,
SPA_VIDEO_FORMAT_RGB,
SPA_VIDEO_FORMAT_RGBA,
SPA_VIDEO_FORMAT_RGBx,
SPA_VIDEO_FORMAT_BGRx,
SPA_VIDEO_FORMAT_YUY2,
SPA_VIDEO_FORMAT_I420),
SPA_FORMAT_VIDEO_size, SPA_POD_CHOICE_RANGE_Rectangle(
&SPA_RECTANGLE(320, 240),
&SPA_RECTANGLE(1, 1),
&SPA_RECTANGLE(4096, 4096)),
SPA_FORMAT_VIDEO_framerate, SPA_POD_CHOICE_RANGE_Fraction(
&SPA_FRACTION(25, 1),
&SPA_FRACTION(0, 1),
&SPA_FRACTION(1000, 1)));
pw_stream_connect(data.stream,
PW_DIRECTION_INPUT,
PW_ID_ANY,
PW_STREAM_FLAG_AUTOCONNECT |
PW_STREAM_FLAG_MAP_BUFFERS,
params, 1);
pw_main_loop_run(data.loop);
pw_stream_destroy(data.stream);
pw_main_loop_destroy(data.loop);
return 0;
}
PW_ID_ANY
#define PW_ID_ANY
Definition core.h:66
PW_KEY_MEDIA_TYPE
#define PW_KEY_MEDIA_TYPE
Media.
Definition keys.h:438
PW_KEY_TARGET_OBJECT
#define PW_KEY_TARGET_OBJECT
a target object to link to.
Definition keys.h:503
PW_KEY_MEDIA_ROLE
#define PW_KEY_MEDIA_ROLE
Role: Movie, Music, Camera, Screen, Communication, Game, Notification, DSP, Production,...
Definition keys.h:444
PW_KEY_MEDIA_CATEGORY
#define PW_KEY_MEDIA_CATEGORY
Media Category: Playback, Capture, Duplex, Monitor, Manager.
Definition keys.h:441
pw_log_warn
#define pw_log_warn(...)
Definition log.h:132
pw_main_loop_destroy
void pw_main_loop_destroy(struct pw_main_loop *loop)
Destroy a loop.
Definition main-loop.c:71
pw_main_loop_run
int pw_main_loop_run(struct pw_main_loop *loop)
Run a main loop.
Definition main-loop.c:120
pw_main_loop_new
struct pw_main_loop * pw_main_loop_new(const struct spa_dict *props)
Create a new main loop.
Definition main-loop.c:61
pw_main_loop_get_loop
struct pw_loop * pw_main_loop_get_loop(struct pw_main_loop *loop)
Get the loop implementation.
Definition main-loop.c:94
pw_init
void pw_init(int *argc, char **argv[])
Initialize PipeWire.
Definition pipewire.c:556
PW_DIRECTION_INPUT
#define PW_DIRECTION_INPUT
Definition port.h:48
pw_properties_new
struct pw_properties * pw_properties_new(const char *key,...)
Make a new properties object.
Definition properties.c:85
pw_properties_set
int pw_properties_set(struct pw_properties *properties, const char *key, const char *value)
Set a property value.
Definition properties.c:422
pw_stream_connect
int pw_stream_connect(struct pw_stream *stream, enum pw_direction direction, uint32_t target_id, enum pw_stream_flags flags, const struct spa_pod **params, uint32_t n_params)
Connect a stream for input or output on port_path.
Definition stream.c:1904
pw_stream_dequeue_buffer
struct pw_buffer * pw_stream_dequeue_buffer(struct pw_stream *stream)
Get a buffer that can be filled for playback streams or consumed for capture streams.
Definition stream.c:2425
pw_stream_queue_buffer
int pw_stream_queue_buffer(struct pw_stream *stream, struct pw_buffer *buffer)
Submit a buffer for playback or recycle a buffer for capture.
Definition stream.c:2453
pw_stream_new_simple
struct pw_stream * pw_stream_new_simple(struct pw_loop *loop, const char *name, struct pw_properties *props, const struct pw_stream_events *events, void *data)
Definition stream.c:1603
PW_VERSION_STREAM_EVENTS
#define PW_VERSION_STREAM_EVENTS
Definition stream.h:345
pw_stream_destroy
void pw_stream_destroy(struct pw_stream *stream)
Destroy a stream.
Definition stream.c:1697
PW_STREAM_FLAG_MAP_BUFFERS
@ PW_STREAM_FLAG_MAP_BUFFERS
mmap the buffers except DmaBuf that is not explicitly marked as mappable.
Definition stream.h:393
PW_STREAM_FLAG_AUTOCONNECT
@ PW_STREAM_FLAG_AUTOCONNECT
try to automatically connect this stream
Definition stream.h:388
spa_debug_type_find_name
static const char * spa_debug_type_find_name(const struct spa_type_info *info, uint32_t type)
Definition types.h:53
spa_format_parse
static int spa_format_parse(const struct spa_pod *format, uint32_t *media_type, uint32_t *media_subtype)
Definition format-utils.h:27
spa_type_video_format
static const struct spa_type_info spa_type_video_format[]
Definition raw-types.h:29
spa_format_video_raw_parse
static int spa_format_video_raw_parse(const struct spa_pod *format, struct spa_video_info_raw *info)
Definition raw-utils.h:27
SPA_MEDIA_TYPE_video
@ SPA_MEDIA_TYPE_video
Definition format.h:28
SPA_PARAM_Format
@ SPA_PARAM_Format
configured format as SPA_TYPE_OBJECT_Format
Definition param.h:34
SPA_PARAM_EnumFormat
@ SPA_PARAM_EnumFormat
available formats as SPA_TYPE_OBJECT_Format
Definition param.h:33
SPA_FORMAT_VIDEO_framerate
@ SPA_FORMAT_VIDEO_framerate
frame rate (Fraction)
Definition format.h:124
SPA_FORMAT_mediaType
@ SPA_FORMAT_mediaType
media type (Id enum spa_media_type)
Definition format.h:93
SPA_FORMAT_VIDEO_size
@ SPA_FORMAT_VIDEO_size
size (Rectangle)
Definition format.h:123
SPA_FORMAT_VIDEO_format
@ SPA_FORMAT_VIDEO_format
video format (Id enum spa_video_format)
Definition format.h:120
SPA_FORMAT_mediaSubtype
@ SPA_FORMAT_mediaSubtype
media subtype (Id enum spa_media_subtype)
Definition format.h:94
SPA_MEDIA_SUBTYPE_raw
@ SPA_MEDIA_SUBTYPE_raw
Definition format.h:38
SPA_VIDEO_FORMAT_YUY2
@ SPA_VIDEO_FORMAT_YUY2
Definition raw.h:49
SPA_VIDEO_FORMAT_RGBA
@ SPA_VIDEO_FORMAT_RGBA
Definition raw.h:56
SPA_VIDEO_FORMAT_RGBx
@ SPA_VIDEO_FORMAT_RGBx
Definition raw.h:52
SPA_VIDEO_FORMAT_BGRx
@ SPA_VIDEO_FORMAT_BGRx
Definition raw.h:53
SPA_VIDEO_FORMAT_I420
@ SPA_VIDEO_FORMAT_I420
Definition raw.h:47
SPA_VIDEO_FORMAT_RGB
@ SPA_VIDEO_FORMAT_RGB
Definition raw.h:60
SPA_POD_CHOICE_ENUM_Id
#define SPA_POD_CHOICE_ENUM_Id(n_vals,...)
Definition vararg.h:51
SPA_POD_CHOICE_RANGE_Fraction
#define SPA_POD_CHOICE_RANGE_Fraction(def, min, max)
Definition vararg.h:115
SPA_POD_Id
#define SPA_POD_Id(val)
Definition vararg.h:49
SPA_POD_BUILDER_INIT
#define SPA_POD_BUILDER_INIT(buffer, size)
Definition builder.h:62
SPA_POD_CHOICE_RANGE_Rectangle
#define SPA_POD_CHOICE_RANGE_Rectangle(def, min, max)
Definition vararg.h:106
spa_pod_builder_add_object
#define spa_pod_builder_add_object(b, type, id,...)
Definition builder.h:659
SPA_TYPE_OBJECT_Format
@ SPA_TYPE_OBJECT_Format
Definition type.h:76
SPA_FRACTION
#define SPA_FRACTION(num, denom)
Definition defs.h:122
SPA_RECTANGLE
#define SPA_RECTANGLE(width, height)
Definition defs.h:101
pipewire.h
pipewire/pipewire.h
pw_buffer
a buffer structure obtained from pw_stream_dequeue_buffer().
Definition stream.h:196
pw_buffer::buffer
struct spa_buffer * buffer
the spa buffer
Definition stream.h:197
pw_main_loop
A main loop object.
pw_properties
Definition properties.h:33
pw_stream_events
Events for a stream.
Definition stream.h:343
pw_stream_events::param_changed
void(* param_changed)(void *data, uint32_t id, const struct spa_pod *param)
when a parameter changed
Definition stream.h:359
pw_stream
spa_buffer
A Buffer.
Definition buffer.h:94
spa_buffer::datas
struct spa_data * datas
array of data members
Definition buffer.h:98
spa_chunk::size
uint32_t size
size of valid data.
Definition buffer.h:47
spa_data::chunk
struct spa_chunk * chunk
valid chunk of memory
Definition buffer.h:90
spa_data::data
void * data
optional data pointer
Definition buffer.h:89
spa_pod_builder
Definition builder.h:53
spa_pod
Definition pod.h:43
spa_pod::type
uint32_t type
Definition pod.h:45
spa_video_info
Definition format.h:27
types.h
spa/debug/types.h

Save as tutorial5.c and compile with:

    gcc -Wall tutorial5.c -o tutorial5 -lm $(pkg-config --cflags --libs libpipewire-0.3)

Most of the application is structured like Tutorial - Part 4: Playing A Tone.

We create a stream object with different properties to make it a Camera Video Capture stream.

props = pw_properties_new(PW_KEY_MEDIA_TYPE, "Video",
PW_KEY_MEDIA_CATEGORY, "Capture",
PW_KEY_MEDIA_ROLE, "Camera",
NULL);
if (argc > 1)
pw_properties_set(props, PW_KEY_TARGET_OBJECT, argv[1]);
data.stream = pw_stream_new_simple(
pw_main_loop_get_loop(data.loop),
"video-capture",
props,
&stream_events,
&data);

We also optionally allow the user to pass the name of the target node where the session manager is supposed to connect the node. The user may also give the value of the unique target node serial (`PW_KEY_OBJECT_SERIAL`) as the value.

In addition to the `process` event, we are also going to listen to a new event, `param_changed`:

static const struct pw_stream_events stream_events = {
PW_VERSION_STREAM_EVENTS,
.param_changed = on_param_changed,
.process = on_process,
};

Because we capture a stream of a wide range of different video formats and resolutions, we have to describe our accepted formats in a different way:

const struct spa_pod *params[1];
uint8_t buffer[1024];
struct spa_pod_builder b = SPA_POD_BUILDER_INIT(buffer, sizeof(buffer));
params[0] = spa_pod_builder_add_object(&b,
SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat,
SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video),
SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw),
SPA_FORMAT_VIDEO_format, SPA_POD_CHOICE_ENUM_Id(7,
SPA_VIDEO_FORMAT_RGB,
SPA_VIDEO_FORMAT_RGB,
SPA_VIDEO_FORMAT_RGBA,
SPA_VIDEO_FORMAT_RGBx,
SPA_VIDEO_FORMAT_BGRx,
SPA_VIDEO_FORMAT_YUY2,
SPA_VIDEO_FORMAT_I420),
SPA_FORMAT_VIDEO_size, SPA_POD_CHOICE_RANGE_Rectangle(
&SPA_RECTANGLE(320, 240),
&SPA_RECTANGLE(1, 1),
&SPA_RECTANGLE(4096, 4096)),
SPA_FORMAT_VIDEO_framerate, SPA_POD_CHOICE_RANGE_Fraction(
&SPA_FRACTION(25, 1),
&SPA_FRACTION(0, 1),
&SPA_FRACTION(1000, 1)));

This is using a `struct spa_pod_builder` to make a `struct spa_pod *` object in the buffer array on the stack. The parameter is of type `SPA_PARAM_EnumFormat` which means that it enumerates the possible formats for this stream.

In this example we use the builder to create some `CHOICE` entries for the format properties.

We have an enumeration of formats, we need to first give the amount of enumerations that follow, then the default (preferred) value, followed by alternatives in order of preference:

SPA_FORMAT_VIDEO_format, SPA_POD_CHOICE_ENUM_Id(7,
SPA_VIDEO_FORMAT_RGB, /* default */
SPA_VIDEO_FORMAT_RGB, /* alternative 1 */
SPA_VIDEO_FORMAT_RGBA, /* alternative 2 */
SPA_VIDEO_FORMAT_RGBx, /* .. etc.. */
SPA_VIDEO_FORMAT_BGRx,
SPA_VIDEO_FORMAT_YUY2,
SPA_VIDEO_FORMAT_I420),

We also have a `RANGE` of values for the size. We need to give a default (preferred) size and then a min and max value:

SPA_FORMAT_VIDEO_size, SPA_POD_CHOICE_RANGE_Rectangle(
&SPA_RECTANGLE(320, 240), /* default */
&SPA_RECTANGLE(1, 1), /* min */
&SPA_RECTANGLE(4096, 4096)), /* max */

We have something similar for the framerate.

Note that there are other video parameters that we don't specify here. This means that we don't have any restrictions for their values.

See SPA POD for more information about how to make these POD objects.

Now we're ready to connect the stream and run the main loop:

pw_stream_connect(data.stream,
PW_DIRECTION_INPUT,
PW_ID_ANY,
PW_STREAM_FLAG_AUTOCONNECT |
PW_STREAM_FLAG_MAP_BUFFERS,
params, 1);
pw_main_loop_run(data.loop);
spa_pod_builder::data
void * data
Definition builder.h:54

To connect we specify that we have a `PW_DIRECTION_INPUT` stream. The third argument is always `PW_ID_ANY`.

We're setting the `PW_STREAM_FLAG_AUTOCONNECT` flag to make an automatic connection to a suitable camera and `PW_STREAM_FLAG_MAP_BUFFERS` to let the stream mmap the data for us.

And last we pass the extra parameters for our stream. Here we only have the allowed formats (`SPA_PARAM_EnumFormat`).

Running the mainloop will start the connection and negotiation process. First our `param_changed` event will be called with the format that was negotiated between our stream and the camera. This is always something that is compatible with what we enumerated in the EnumFormat param when we connected.

Let's take a look at how we can parse the format in the `param_changed` event:

static void on_param_changed(void *userdata, uint32_t id, const struct spa_pod *param)
{
struct data *data = userdata;
if (param == NULL || id != SPA_PARAM_Format)
return;

First check if there is a param. A NULL param means that it is cleared. The ID of the param tells you what param it is. We are only interested in Format param (`SPA_PARAM_Format`).

We can parse the media type and subtype as below and ensure that it is of the right type. In our example this will always be true but when your EnumFormat contains different media types or subtypes, this is how you can parse them:

if (spa_format_parse(param,
&data->format.media_type,
&data->format.media_subtype) < 0)
return;
if (data->format.media_type != SPA_MEDIA_TYPE_video ||
data->format.media_subtype != SPA_MEDIA_SUBTYPE_raw)
return;

For the `video/raw` media type/subtype there is a utility function to parse out the values into a `struct spa_video_info`. This makes it easier to deal with.

if (spa_format_video_raw_parse(param, &data->format.info.raw) < 0)
return;
printf("got video format:\n");
printf(" format: %d (%s)\n", data->format.info.raw.format,
spa_debug_type_find_name(spa_type_video_format,
data->format.info.raw.format));
printf(" size: %dx%d\n", data->format.info.raw.size.width,
data->format.info.raw.size.height);
printf(" framerate: %d/%d\n", data->format.info.raw.framerate.num,
data->format.info.raw.framerate.denom);
}

In this example we dump the video size and parameters but in a real playback or capture application you might want to set up the screen or encoder to deal with the format.

After negotiation, the process function is called for each new frame. Check out Tutorial - Part 4: Playing A Tone for another example.

static void on_process(void *userdata)
{
struct data *data = userdata;
struct pw_buffer *b;
struct spa_buffer *buf;
if ((b = pw_stream_dequeue_buffer(data->stream)) == NULL) {
pw_log_warn("out of buffers: %m");
return;
}
buf = b->buffer;
if (buf->datas[0].data == NULL)
return;
printf("got a frame of size %d\n", buf->datas[0].chunk->size);
pw_stream_queue_buffer(data->stream, b);
}

In a real playback application, one would do something with the data, like copy it to the screen or encode it into a file.

Tutorial - Part 4: Playing A Tone | Index | Tutorial - Part 6: Binding Objects