Code Bank
0%

GStreamer换GstSample问题分析

نُشر في عُدل في

问题

现存如下GStreamer管道：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
typedef struct g_pipeline0_style {
	GstElement *pipeline;

	GstElement *src;
	GstElement *yuy2_filter;
	GstElement *v4l2convert;
	GstElement *rgb_filter;
	GstElement *appsink;
} g_pipeline0_style;

typedef struct g_pipeline1_style {
	GstElement *pipeline;

	GstElement *appsrc;
	GstElement *rgb_filter;
	GstElement *waylandsink;
} g_pipeline1_style;

void init_gstreamer(g_pipeline0_style *pipeline0, g_pipeline1_style *pipeline1) {
	g_assert(pipeline0 && pipeline1);

	// 创建
	pipeline0->pipeline = gst_pipeline_new("pipeline0");
	pipeline0->src = gst_element_factory_make(GST_SRC, "src");
	pipeline0->yuy2_filter = gst_element_factory_make("capsfilter", "videoconvert");
	pipeline0->v4l2convert = gst_element_factory_make("v4l2convert", "v4l2convert");
	pipeline0->rgb_filter = gst_element_factory_make("capsfilter", "rgb_filter");
	pipeline0->appsink = gst_element_factory_make("appsink", "appsink");

	pipeline1->pipeline = gst_pipeline_new("pipeline1");
	pipeline1->appsrc = gst_element_factory_make("appsrc", "appsrc");
	pipeline1->rgb_filter = gst_element_factory_make("capsfilter", "rgb_filter");
	pipeline1->waylandsink = gst_element_factory_make("waylandsink", "waylandsink");

	g_assert(pipeline0->pipeline && pipeline0->src && pipeline0->yuy2_filter &&
		pipeline0->v4l2convert && pipeline0->rgb_filter && pipeline0->appsink &&
		pipeline1->pipeline && pipeline1->appsrc && pipeline1->rgb_filter &&
		pipeline1->waylandsink);

	// 设置
	GstCaps *caps_yuy2 = gst_caps_new_simple("video/x-raw",
		"width", G_TYPE_INT, DEFAULT_WIDTH,
		"height", G_TYPE_INT, DEFAULT_HEIGHT,
		"format", G_TYPE_STRING, "YUY2",
		"framerate", GST_TYPE_FRACTION, 30, 1,
		NULL);
	g_assert(caps_yuy2);
	g_object_set(G_OBJECT(pipeline0->yuy2_filter), "caps", caps_yuy2, NULL);
	gst_caps_unref(caps_yuy2);

	g_object_set(pipeline0->v4l2convert, "disable-passthrough", TRUE, NULL);
	gst_util_set_object_arg(G_OBJECT(pipeline0->v4l2convert), "output-io-mode", "dmabuf-import");
	gst_util_set_object_arg(G_OBJECT(pipeline0->v4l2convert), "capture-io-mode", "dmabuf");

	GstCaps *caps_rgb0 = gst_caps_new_simple("video/x-raw",
		"width", G_TYPE_INT, DEFAULT_WIDTH,
		"height", G_TYPE_INT, DEFAULT_HEIGHT,
		"format", G_TYPE_STRING, "RGB",
		"framerate", GST_TYPE_FRACTION, 30, 1,
		NULL);
	g_assert(caps_rgb0);
	g_object_set(G_OBJECT(pipeline0->rgb_filter), "caps", caps_rgb0, NULL);
	gst_caps_unref(caps_rgb0);

	GstAppSinkCallbacks app_sink_callback = { NULL, NULL, on_new_sample };
	gst_app_sink_set_callbacks(GST_APP_SINK(pipeline0->appsink), &app_sink_callback,
		pipeline1->appsrc, NULL);

	GstCaps *caps_rgb1 = gst_caps_new_simple("video/x-raw",
		"width", G_TYPE_INT, DEFAULT_WIDTH,
		"height", G_TYPE_INT, DEFAULT_HEIGHT,
		"format", G_TYPE_STRING, "RGB",
		"framerate", GST_TYPE_FRACTION, 30, 1,
		NULL);
	g_assert(caps_rgb1);
	g_object_set(G_OBJECT(pipeline1->rgb_filter), "caps", caps_rgb1, NULL);
	gst_caps_unref(caps_rgb1);

	g_object_set(G_OBJECT(pipeline1->waylandsink), "sync", FALSE, NULL);

	// 将元素添加到管道中
	gst_bin_add_many(GST_BIN(pipeline0->pipeline), pipeline0->src, pipeline0->yuy2_filter,
		pipeline0->v4l2convert, pipeline0->rgb_filter, pipeline0->appsink, NULL);
	gst_bin_add_many(GST_BIN(pipeline1->pipeline), pipeline1->appsrc, pipeline1->rgb_filter,
		pipeline1->waylandsink, NULL);

	// 连接元素
	gst_bin_link_many(pipeline0->src, pipeline0->yuy2_filter, pipeline0->v4l2convert,
		pipeline0->rgb_filter, pipeline0->appsink, NULL);
	gst_bin_link_many(pipeline1->appsrc, pipeline1->rgb_filter, pipeline1->waylandsink, NULL);
}

两个管道间使用如下函数进行链接：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
GstFlowReturn on_new_sample(GstAppSink *sink, gpointer user_data) {
	GstFlowReturn ret = GST_FLOW_ERROR;

	GstSample *sample = gst_app_sink_pull_sample(sink);
	GstAppSrc *app_src = (GstAppSrc *)user_data;
	g_assert(sample && app_src);

	if(!gst_sample_is_writable(sample)) {
		sample = gst_sample_make_writable(sample);
	}

	g_assert(gst_sample_is_writable(sample));

	GstBuffer *buffer = gst_sample_get_buffer(sample);
	g_assert(buffer);

	if(!gst_buffer_is_writable(buffer)) {
		buffer = gst_buffer_make_writable(buffer);
	}

	g_assert(gst_buffer_is_writable(buffer));

	gboolean ok;
	GstMapInfo info;
	ok = gst_buffer_map(buffer, &info, GST_MAP_READWRITE);
	g_assert(ok);

	GstCaps *caps = gst_sample_get_caps(sample);
	gint width, height;
	GstStructure *structure = gst_caps_get_structure(caps, 0);
	gst_structure_get_int(structure, "width", &width);
	gst_structure_get_int(structure, "height", &height);

	do_something(info.data, height, width);

	gst_buffer_unmap(buffer, &info);

	gst_sample_set_buffer(sample, buffer);

	ret = gst_app_src_push_sample(GST_APP_SRC(app_src), sample);
	gst_sample_unref(sample);
	return ret;
}

但在运行时报错：

1
2
3
GStreamer-Wayland:ERROR:../gst-plugins-bad-1.22.0/gst-libs/gst/wayland/gstwlbuffer.c:178:gstmemory_disposed: assertion failed: (!priv->used_by_compositor)
Bail out! GStreamer-Wayland:ERROR:../gst-plugins-bad-1.22.0/gst-libs/gst/wayland/gstwlbuffer.c:178:gstmemory_disposed: assertion failed: (!priv->used_by_compositor)
Aborted (core dumped)

查看日志，发现第一条错误如下：

1
ERROR waylandsink gstwaylandsink.c:1181:gst_wayland_sink_show_frame:<waylandsink> buffer buffer: 0x55a234bbf0, pts 0:00:00.000000000, dts 0:00:00.000000000, dur 0:00:00.033333333, size 3072000, offset none, offset_end none, flags 0x40 cannot have a wl_buffer

原因

主要是因为缓冲区类型和数据类型不匹配导致的。WaylandSink仅支持DMA Buffer携带的RGB数据，但在复制后，Buffer类型变成了Share Memory Buffer，这是WaylandSink不支持的，因此报错。

解决

虽然从理论上来说将复制出的Buffer转换为DMA Buffer就可以解决该问题。但笔者工作较忙没有进行论证。

还有一种妥协性的解决方法，就是去除pipeline0->v4l2convert和pipeline0->rgb_filter，将pipeline1->rgb_filter类型转换为YUY2，这样到达WaylandSink的就是YUY2，而WaylandSink支持携带YUY2数据的Share Memory Buffer。