Clever Geek Handbook

Cómo desarrollar un análogo de Zoom para TV Box en RDK y Linux. Comprender el marco de GStreamer

Escenarios: cómo utilizar una aplicación de videoconferencia en SmartTV y decodificadores
Escenarios: cómo utilizar una aplicación de videoconferencia en SmartTV y decodificadores

La pandemia de COVID-19 se ha convertido en un catalizador de nuevos servicios útiles. Por ejemplo, Zoom se ha vuelto tan exitoso que superó a IBM en valor este mes. Este ejemplo nos inspiró y decidimos ir aún más lejos: ¿qué pasaría si implementamos conferencias en línea en decodificadores y televisores inteligentes para comunicarnos no solo en el trabajo, sino para organizar reuniones remotas en el sofá con amigos? Pero luego pueden gritar juntos en el fútbol, ​​ver una película o practicar deportes bajo la supervisión de un entrenador. 

- , - Linux/Android RDK. « Zoom» Smart TV. GStreamer. , .

- . , desktop-, , , embedded- .

, -:

  1. . STB-   ARM-, , / . , — .

  2. . Android, — RDK, — Linux . . desktop-. .

  3. . Ethernet wifi. / — .

  4. . .

  5. .

. Zoom - :

  • /

  • /

:

Arquitectura de la aplicación de videoconferencia de Smart TV
Smart TV

GStreamer, .. .

/  

1) GStreamer

, . , 30 640x480. , RGB24 :

640 480 3 30 = 27 648 000 , .. 26 , .

— - . , , GStreamer. ? :

  1. Linux Android.

  2. RDK Gstreamer / -.

  3. , . FFmpeg, , - GStreamer’.

  4. (pipeline). / , , .

  5. API /C++ .

  6. /, OpenMAX API — -.

2) GStreamer  

, , .  GStreamer , :

gst-inspect-1.0 , , , .

gst-launch-1.0 (pipeline).

GStreamer , , source, sink-. source — , ,  (sink) — , , ( RTP).

. mp4-:

gst-launch-1.0 filesrc location=file.mp4 ! qtdemux ! h264parse ! avdec_h264 ! videoconvert ! autovideosink

mp4-, mp4 — qtdemux, h264, , , , .

autovideosink filesink .

3) GStreamer C/C++ API.

, gst-launch-1.0, , . : (pipeline), GStreamer glib-.

filesrc filesink — «GStreamer: ». H264-.

GStreamer-

gstinit (NULL, NULL);

, 

gst_debug_set_active(TRUE);
gst_debug_set_default_threshold(GST_LEVEL_LOG);

: , gstinit .

event-loop, :

GMainLoop *loop;
loop = g_main_loop_new (NULL, FALSE);

:

, GstElement:

GstElement *pipeline, *source, *demuxer, *parser, *decoder, *conv, *sink;

pipeline = gst_pipeline_new ("video-decoder");
source   = gst_element_factory_make ("filesrc",       "file-source");
demuxer  = gst_element_factory_make ("qtdemux",      "h264-demuxer");
parser   = gst_element_factory_make ("h264parse",      "h264-parser");
decoder  = gst_element_factory_make ("avdec_h264",     "h264-decoder");
conv     = gst_element_factory_make ("videoconvert",  "converter");
sink     = gst_element_factory_make ("appsink", "video-output");

gst_element_factory_make, , — GStreamer, , , .

, , gst_element_factory_make NULL.

if (!pipeline || !source || !demuxer || !parser || !decoder || !conv || !sink) {
		//     - 
		return;
}

location gob_ject_set:

gob_ject_set (G_OBJECT (source), "location", argv[1], NULL);

.

GStreamer, bus_call:

GstBus *bus;
	guint bus_watch_id;
	bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
	bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
	gst_object_unref (bus);

gst_object_unref .

:

static gboolean
bus_call (GstBus     *bus,
          GstMessage *msg,
          gpointer    data)
{
  GMainLoop *loop = (GMainLoop *) data;

  switch (GST_MESSAGE_TYPE (msg)) {

    case GST_MESSAGE_EOS:
      LOGI ("End of stream\n");
      g_main_loop_quit (loop);
      break;

    case GST_MESSAGE_ERROR: {
      gchar  *debug;
      GError *error;

      gst_message_parse_error (msg, &error, &debug);
      g_free (debug);

      LOGE ("Error: %s\n", error->message);
      g_error_free (error);

      g_main_loop_quit (loop);
      break;
    }
    default:
      break;
  }

  return TRUE;
}

: , gst-launch. , , :

gst_bin_add_many (GST_BIN (pipeline), source, demuxer, parser, decoder, conv, sink, NULL);
gst_element_link_many (source, demuxer, parser, decoder, conv, sink, NULL);

, , (autovideosink) :

  gst_element_link (source, demuxer);
  gst_element_link_many (parser, decoder, conv, sink, NULL);
  g_signal_connect (demuxer, "pad-added", G_CALLBACK (on_pad_added), parser);

static void
on_pad_added (GstElement *element,
              GstPad     *pad,
              gpointer    data)
{
  GstPad *sinkpad;
  GstElement *decoder = (GstElement *) data;

  /* We can now link this pad with the sink pad */
  g_print ("Dynamic pad created, linking demuxer/decoder\n");

  sinkpad = gst_element_get_static_pad (decoder, "sink");

  gst_pad_link (pad, sinkpad);

  gst_object_unref (sinkpad);
}

, .

, , :

gst_element_set_state (pipeline, GST_STATE_PLAYING);

event-loop:

g_main_loop_run (loop);

:

gst_element_set_state (pipeline, GST_STATE_NULL);
  gst_object_unref (GST_OBJECT (pipeline));
  g_source_remove (bus_watch_id);
  g_main_loop_unref (loop);

4) .

, — , .

gst_element_factory_find, , factory :

if(gst_element_factory_find("omxh264dec"))
		decoder  = gst_element_factory_make ("omxh264dec",     "h264-decoder");
	else
		decoder  = gst_element_factory_make ("avdec_h264",     "h264-decoder");

OMX RDK, .

, , GstElement ( ):

gst_plugin_feature_get_name(gst_element_get_factory(encoder))

.

5)

, . YUV, RGB.

YUYV. , GStreamer, I420. , gl-, I420-. . , .

GStreamer’ , , - . 

 

1)

  . , , filesrc filesink .

appsrc / appsink. - . 

, ? , . , I420. , ? ?

need-data, :

g_signal_connect (source, "need-data", G_CALLBACK (encoder_cb_need_data), NULL);

:

encoder_cb_need_data (GstElement *appsrc,
          guint       unused_size,
          gpointer    user_data)
{
  GstBuffer *buffer;
  GstFlowReturn ret;
  GstMapInfo map;

   int size;
   uint8_t* image;
  // get image

  buffer = gst_buffer_new_allocate (NULL, size, NULL);
  gst_buffer_map (buffer, &map, GST_MAP_WRITE);
  
  memcpy((guchar *)map.data, image,  gst_buffer_get_size( buffer ) );
  gst_buffer_unmap(buffer, &map);

  g_signal_emit_by_name (appsrc, "push-buffer", buffer, &ret);
  gst_buffer_unref(buffer);
}

image — , , I420.

gst_buffer_new_allocate , .

gst_buffer_map , memcpy, . 

, , GStream’ , .

: gst_buffer_unmap, gst_buffer_unref. . , , , .

, , : caps .

need-data:

	g_object_set (G_OBJECT (source),
        "stream-type", 0,
        "format", GST_FORMAT_TIME, NULL);

	g_object_set (G_OBJECT (source), "caps",
		gst_caps_new_simple ("video/x-raw",
					"format", G_TYPE_STRING, "I420",
					"width", G_TYPE_INT, 640,
					"height", G_TYPE_INT, 480,
					"framerate", GST_TYPE_FRACTION, 30, 1,
		NULL), 
	NULL);

GstElement, g_object_set.

, caps — . , appsrc I420 c 640x480 30 .

, , . , GStreamer - need-data .

, . 

2)

, .

sink pad:

	GstPad *pad = gst_element_get_static_pad (sink, "sink");
  	gst_pad_add_probe  (pad, GST_PAD_PROBE_TYPE_BUFFER, encoder_cb_have_data, NULL, NULL);
  	gst_object_unref (pad);

sink pad — GST_PAD_PROBE_TYPE_BUFFER, — sink pad.

static GstPadProbeReturn
encoder_cb_have_data (GstPad * pad,
                        GstPadProbeInfo * info,
                        gpointer user_data) {

  GstBuffer *buf = gst_pad_probe_info_get_buffer (info);
  GstMemory *bufMem = gst_buffer_get_memory(buf, 0);
  GstMapInfo bufInfo;
  gst_memory_map(bufMem, &bufInfo, GST_MAP_READ);

  // bufInfo.data, bufInfo.size

  gst_memory_unmap(bufMem, &bufInfo);

  return GST_PAD_PROBE_OK;
}

. . GstBuffer, , gst_buffer_get_memory 0 ( ). , , gst_memory_map, bufInfo.data bufInfo.size.

— .

, Smart TV — Zoom -: , / GStreamer, / .

. — — embedded- RDK, Linux Android. — , .

Esta idea con un servicio de videoconferencia a través de Smart TV se puede desarrollar aún más, tanto en términos de soluciones de ingeniería como de escenarios de su uso. Así que comparte tus pensamientos en los comentarios. 



More articles:


All Articles