So entwickeln Sie ein Zoom-Analogon für TV-Boxen unter RDK und Linux. Grundlegendes zum GStreamer-Framework

Die COVID-19-Pandemie ist zu einem Katalysator für nützliche neue Dienste geworden. Zum Beispiel ist Zoom so erfolgreich geworden, dass es IBM in diesem Monat an Wert überholt hat. Dieses Beispiel hat uns inspiriert und wir haben uns entschlossen, noch weiter zu gehen: Was ist, wenn wir Online-Konferenzen auf Set-Top-Boxen und Smart-TVs durchführen, um nicht nur bei der Arbeit zu kommunizieren, sondern auch entfernte Treffen auf der Couch mit Freunden zu arrangieren? Aber dann können Sie gemeinsam Fußball schreien, einen Film schauen oder unter der Aufsicht eines Trainers Sport treiben. 

- , - Linux/Android RDK. « Zoom» Smart TV. GStreamer. , .

- . , desktop-, , , embedded- .

, -:

1. . STB-   ARM-, , / . , — .

2. . Android, — RDK, — Linux . . desktop-. .

3. . Ethernet wifi. / — .

4. . .

5. .

. Zoom - :

• /

• /

:

GStreamer, .. .

/  

1) GStreamer

, . , 30 640x480. , RGB24 :

640 480 3 30 = 27 648 000 , .. 26 , .

— - . , , GStreamer. ? :

1. Linux Android.

2. RDK Gstreamer / -.

3. , . FFmpeg, , - GStreamer’.

4. (pipeline). / , , .

5. API /C++ .

6. /, OpenMAX API — -.

2) GStreamer  

, , .  GStreamer , :

gst-inspect-1.0 , , , .

gst-launch-1.0 (pipeline).

GStreamer , , source, sink-. source — , ,  (sink) — , , ( RTP).

. mp4-:

gst-launch-1.0 filesrc location=file.mp4 ! qtdemux ! h264parse ! avdec_h264 ! videoconvert ! autovideosink

mp4-, mp4 — qtdemux, h264, , , , .

autovideosink filesink .

3) GStreamer C/C++ API.

, gst-launch-1.0, , . : (pipeline), GStreamer glib-.

filesrc filesink — «GStreamer: ». H264-.

GStreamer-

gstinit (NULL, NULL);

,

gst_debug_set_active(TRUE);
gst_debug_set_default_threshold(GST_LEVEL_LOG);

: , gstinit .

event-loop, :

GMainLoop *loop;
loop = g_main_loop_new (NULL, FALSE);

:

, GstElement:

GstElement *pipeline, *source, *demuxer, *parser, *decoder, *conv, *sink;

pipeline = gst_pipeline_new ("video-decoder");
source   = gst_element_factory_make ("filesrc",       "file-source");
demuxer  = gst_element_factory_make ("qtdemux",      "h264-demuxer");
parser   = gst_element_factory_make ("h264parse",      "h264-parser");
decoder  = gst_element_factory_make ("avdec_h264",     "h264-decoder");
conv     = gst_element_factory_make ("videoconvert",  "converter");
sink     = gst_element_factory_make ("appsink", "video-output");

gst_element_factory_make, , — GStreamer, , , .

, , gst_element_factory_make NULL.

if (!pipeline || !source || !demuxer || !parser || !decoder || !conv || !sink) {
		//     - 
		return;
}

location gob_ject_set:

gob_ject_set (G_OBJECT (source), "location", argv[1], NULL);

.

GStreamer, bus_call:

GstBus *bus;
	guint bus_watch_id;
	bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
	bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
	gst_object_unref (bus);

gst_object_unref .

:

static gboolean
bus_call (GstBus     *bus,
          GstMessage *msg,
          gpointer    data)
{
  GMainLoop *loop = (GMainLoop *) data;

  switch (GST_MESSAGE_TYPE (msg)) {

    case GST_MESSAGE_EOS:
      LOGI ("End of stream\n");
      g_main_loop_quit (loop);
      break;

    case GST_MESSAGE_ERROR: {
      gchar  *debug;
      GError *error;

      gst_message_parse_error (msg, &error, &debug);
      g_free (debug);

      LOGE ("Error: %s\n", error->message);
      g_error_free (error);

      g_main_loop_quit (loop);
      break;
    }
    default:
      break;
  }

  return TRUE;
}

: , gst-launch. , , :

gst_bin_add_many (GST_BIN (pipeline), source, demuxer, parser, decoder, conv, sink, NULL);
gst_element_link_many (source, demuxer, parser, decoder, conv, sink, NULL);

, , (autovideosink) :

  gst_element_link (source, demuxer);
  gst_element_link_many (parser, decoder, conv, sink, NULL);
  g_signal_connect (demuxer, "pad-added", G_CALLBACK (on_pad_added), parser);

static void
on_pad_added (GstElement *element,
              GstPad     *pad,
              gpointer    data)
{
  GstPad *sinkpad;
  GstElement *decoder = (GstElement *) data;

  /* We can now link this pad with the sink pad */
  g_print ("Dynamic pad created, linking demuxer/decoder\n");

  sinkpad = gst_element_get_static_pad (decoder, "sink");

  gst_pad_link (pad, sinkpad);

  gst_object_unref (sinkpad);
}

, .

, , :

gst_element_set_state (pipeline, GST_STATE_PLAYING);

event-loop:

g_main_loop_run (loop);

:

gst_element_set_state (pipeline, GST_STATE_NULL);
  gst_object_unref (GST_OBJECT (pipeline));
  g_source_remove (bus_watch_id);
  g_main_loop_unref (loop);

4) .

, — , .

gst_element_factory_find, , factory :

if(gst_element_factory_find("omxh264dec"))
		decoder  = gst_element_factory_make ("omxh264dec",     "h264-decoder");
	else
		decoder  = gst_element_factory_make ("avdec_h264",     "h264-decoder");

OMX RDK, .

, , GstElement ( ):

gst_plugin_feature_get_name(gst_element_get_factory(encoder))

.

5)

, . YUV, RGB.

YUYV. , GStreamer, I420. , gl-, I420-. . , .

GStreamer’ , , - . 

1)

  . , , filesrc filesink .

appsrc / appsink. - . 

, ? , . , I420. , ? ?

need-data, :

g_signal_connect (source, "need-data", G_CALLBACK (encoder_cb_need_data), NULL);

:

encoder_cb_need_data (GstElement *appsrc,
          guint       unused_size,
          gpointer    user_data)
{
  GstBuffer *buffer;
  GstFlowReturn ret;
  GstMapInfo map;

   int size;
   uint8_t* image;
  // get image

  buffer = gst_buffer_new_allocate (NULL, size, NULL);
  gst_buffer_map (buffer, &map, GST_MAP_WRITE);
  
  memcpy((guchar *)map.data, image,  gst_buffer_get_size( buffer ) );
  gst_buffer_unmap(buffer, &map);

  g_signal_emit_by_name (appsrc, "push-buffer", buffer, &ret);
  gst_buffer_unref(buffer);
}

image — , , I420.

gst_buffer_new_allocate , .

gst_buffer_map , memcpy, . 

, , GStream’ , .

: gst_buffer_unmap, gst_buffer_unref. . , , , .

, , : caps .

need-data:

	g_object_set (G_OBJECT (source),
        "stream-type", 0,
        "format", GST_FORMAT_TIME, NULL);

	g_object_set (G_OBJECT (source), "caps",
		gst_caps_new_simple ("video/x-raw",
					"format", G_TYPE_STRING, "I420",
					"width", G_TYPE_INT, 640,
					"height", G_TYPE_INT, 480,
					"framerate", GST_TYPE_FRACTION, 30, 1,
		NULL), 
	NULL);

GstElement, g_object_set.

, caps — . , appsrc I420 c 640x480 30 .

, , . , GStreamer - need-data .

, . 

2)

, .

sink pad:

	GstPad *pad = gst_element_get_static_pad (sink, "sink");
  	gst_pad_add_probe  (pad, GST_PAD_PROBE_TYPE_BUFFER, encoder_cb_have_data, NULL, NULL);
  	gst_object_unref (pad);

sink pad — GST_PAD_PROBE_TYPE_BUFFER, — sink pad.

static GstPadProbeReturn
encoder_cb_have_data (GstPad * pad,
                        GstPadProbeInfo * info,
                        gpointer user_data) {

  GstBuffer *buf = gst_pad_probe_info_get_buffer (info);
  GstMemory *bufMem = gst_buffer_get_memory(buf, 0);
  GstMapInfo bufInfo;
  gst_memory_map(bufMem, &bufInfo, GST_MAP_READ);

  // bufInfo.data, bufInfo.size

  gst_memory_unmap(bufMem, &bufInfo);

  return GST_PAD_PROBE_OK;
}

. . GstBuffer, , gst_buffer_get_memory 0 ( ). , , gst_memory_map, bufInfo.data bufInfo.size.

— .

, Smart TV — Zoom -: , / GStreamer, / .

. — — embedded- RDK, Linux Android. — , .

Diese Idee mit einem Videokonferenzdienst über Smart TV kann sowohl in Bezug auf technische Lösungen als auch in Bezug auf Szenarien ihrer Verwendung weiterentwickelt werden. Teilen Sie also Ihre Gedanken in den Kommentaren mit.