GStreamer DispTEC Plugin: Difference between revisions

From RidgeRun Developer Wiki
mNo edit summary
 
(10 intermediate revisions by 3 users not shown)
Line 1: Line 1:
= Overview =
<seo title="GStreamer Motion Detection | GStreamer DispTEC | RidgeRun" titlemode="replace" keywords="Motion Detection, DispTEC, GStreamer DispTEC, GStreamer Motion Detection, OpenCV, GstDispTEC, CUDA, GStreamer, GStreamer DispTEC Plugin, DispTEC Tracker Plugin, GstDispTEC Object Tracking" description="Learn about GstDispTEC Motion Detection from the RidgeRun Developer Wiki."></seo>


GstDispTEC is a GStreamer plug-in which integrates DispTEC <ref>http://palvarado.ietec.org/pmwiki/index.php/Proyectos/DispTEC?userlang=en</ref> library's algorithms, making it possible to incorporate their functionality into GStreamer pipelines, such as motion detection in a video sequence or object tracking, showing where the object is located on a frame, even with non-stationary cameras!
<table cellspacing="0">
<tr>
<td><div class="clear; float:right"> __TOC__</div></td>
<td>
<html>
<div id='product-component-0904c5f574e'></div>
    <script type="text/javascript">
    /*<![CDATA[*/


With GstDispTEC, it is possible to add several DispTEC-based elements in the same pipeline to strengthen image analysis, connect your own applications to the pipeline for receiving the analysis data and much more amazing things. Also, the algorithms can be run either on CPU or GPU, making the most of the computational resources and reducing the processing time.
    (function () {
      var scriptURL = 'https://sdks.shopifycdn.com/buy-button/latest/buy-button-storefront.min.js';
      if (window.ShopifyBuy) {
        if (window.ShopifyBuy.UI) {
          ShopifyBuyInit();
        } else {
          loadScript();
        }
      } else {
        loadScript();
      }


== Demonstration ==
      function loadScript() {
        var script = document.createElement('script');
        script.async = true;
        script.src = scriptURL;
        (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(script);
        script.onload = ShopifyBuyInit;
      }


=== Object tracking element ===
      function ShopifyBuyInit() {
        var client = ShopifyBuy.buildClient({
          domain: 'ridgerun1.myshopify.com',
          storefrontAccessToken: 'b0ca98633a82de5d2f63cd51f5af30ac',
        });


<center>
        ShopifyBuy.UI.onReady(client).then(function (ui) {
<embedvideo service="youtube">https://youtu.be/FGl8Sp43JCk</embedvideo>
          ui.createComponent('product', {
</center>
            id: [1718592077895],
            node: document.getElementById('product-component-0904c5f574e'),
            moneyFormat: '%24%7B%7Bamount%7D%7D',
            options: {
  "product": {
    "variantId": "all",
    "width": "240px",
    "contents": {
      "imgWithCarousel": false,
      "variantTitle": false,
      "description": false,
      "buttonWithQuantity": false,
      "quantity": false
    },
    "text": {
      "button": "BUY NOW"
    },
    "styles": {
      "product": {
        "@media (min-width: 601px)": {
          "max-width": "100%",
          "margin-left": "0",
          "margin-bottom": "50px"
        }
      },
      "button": {
        "background-color": "#007493",
        "font-size": "18px",
        "padding-top": "17px",
        "padding-bottom": "17px",
        ":hover": {
          "background-color": "#006884"
        },
        ":focus": {
          "background-color": "#006884"
        }
      },
      "quantityInput": {
        "font-size": "18px",
        "padding-top": "17px",
        "padding-bottom": "17px"
      },
      "compareAt": {
        "font-size": "12px"
      }
    }
  },
  "cart": {
    "contents": {
      "button": true
    },
    "styles": {
      "button": {
        "background-color": "#007493",
        "font-size": "18px",
        "padding-top": "17px",
        "padding-bottom": "17px",
        ":hover": {
          "background-color": "#006884"
        },
        ":focus": {
          "background-color": "#006884"
        }
      },
      "footer": {
        "background-color": "#ffffff"
      }
    }
  },
  "modalProduct": {
    "contents": {
      "img": false,
      "imgWithCarousel": true,
      "variantTitle": false,
      "buttonWithQuantity": true,
      "button": false,
      "quantity": false
    },
    "styles": {
      "product": {
        "@media (min-width: 601px)": {
          "max-width": "100%",
          "margin-left": "0px",
          "margin-bottom": "0px"
        }
      },
      "button": {
        "background-color": "#007493",
        "font-size": "18px",
        "padding-top": "17px",
        "padding-bottom": "17px",
        ":hover": {
          "background-color": "#006884"
        },
        ":focus": {
          "background-color": "#006884"
        }
      },
      "quantityInput": {
        "font-size": "18px",
        "padding-top": "17px",
        "padding-bottom": "17px"
      }
    }
  },
  "toggle": {
    "styles": {
      "toggle": {
        "background-color": "#007493",
        ":hover": {
          "background-color": "#006884"
        },
        ":focus": {
          "background-color": "#006884"
        }
      },
      "count": {
        "font-size": "18px"
      }
    }
  },
  "productSet": {
    "styles": {
      "products": {
        "@media (min-width: 601px)": {
          "margin-left": "-20px"
        }
      }
    }
  }
}
          });
        });
      }
    })();
    /*]]>*/
</script>
</html>
<td><center>
<html>
<div id='product-component-c61a89404de'></div>
    <script type="text/javascript">
    /*<![CDATA[*/


=== Motion detection element ===
    (function () {
      var scriptURL = 'https://sdks.shopifycdn.com/buy-button/latest/buy-button-storefront.min.js';
      if (window.ShopifyBuy) {
        if (window.ShopifyBuy.UI) {
          ShopifyBuyInit();
        } else {
          loadScript();
        }
      } else {
        loadScript();
      }


<center>
      function loadScript() {
<embedvideo service="youtube">https://youtu.be/4zTZWsdi0Qg</embedvideo>
        var script = document.createElement('script');
</center>
        script.async = true;
        script.src = scriptURL;
        (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(script);
        script.onload = ShopifyBuyInit;
      }


= Architecture =
      function ShopifyBuyInit() {
        var client = ShopifyBuy.buildClient({
          domain: 'ridgerun1.myshopify.com',
          storefrontAccessToken: 'b0ca98633a82de5d2f63cd51f5af30ac',
        });


GstDispTEC has been created based on DispTEC library. However, GstDispTEC is independent of the DispTEC's dependencies and it can be easily integrated whichever library that DispTEC is based on (OpenCV, LTI-lib, etc). This is achieved by an ''Abstraction layer'' just on the base of GstDispTEC plug-in, which can adapt it to several DispTEC variations.
        ShopifyBuy.UI.onReady(client).then(function (ui) {
          ui.createComponent('product', {
            id: [1718577266759],
            node: document.getElementById('product-component-c61a89404de'),
            moneyFormat: '%24%7B%7Bamount%7D%7D',
            options: {
  "product": {
    "variantId": "all",
    "width": "240px",
    "contents": {
      "imgWithCarousel": false,
      "variantTitle": false,
      "description": false,
      "buttonWithQuantity": false,
      "quantity": false
    },
    "text": {
      "button": "BUY NOW"
    },
    "styles": {
      "product": {
        "@media (min-width: 601px)": {
          "max-width": "100%",
          "margin-left": "0",
          "margin-bottom": "50px"
        }
      },
      "button": {
        "background-color": "#007493",
        "font-size": "18px",
        "padding-top": "17px",
        "padding-bottom": "17px",
        ":hover": {
          "background-color": "#006884"
        },
        ":focus": {
          "background-color": "#006884"
        }
      },
      "quantityInput": {
        "font-size": "18px",
        "padding-top": "17px",
        "padding-bottom": "17px"
      },
      "compareAt": {
        "font-size": "12px"
      }
    }
  },
  "cart": {
    "contents": {
      "button": true
    },
    "styles": {
      "button": {
        "background-color": "#007493",
        "font-size": "18px",
        "padding-top": "17px",
        "padding-bottom": "17px",
        ":hover": {
          "background-color": "#006884"
        },
        ":focus": {
          "background-color": "#006884"
        }
      },
      "footer": {
        "background-color": "#ffffff"
      }
    }
  },
  "modalProduct": {
    "contents": {
      "img": false,
      "imgWithCarousel": true,
      "variantTitle": false,
      "buttonWithQuantity": true,
      "button": false,
      "quantity": false
    },
    "styles": {
      "product": {
        "@media (min-width: 601px)": {
          "max-width": "100%",
          "margin-left": "0px",
          "margin-bottom": "0px"
        }
      },
      "button": {
        "background-color": "#007493",
        "font-size": "18px",
        "padding-top": "17px",
        "padding-bottom": "17px",
        ":hover": {
          "background-color": "#006884"
        },
        ":focus": {
          "background-color": "#006884"
        }
      },
      "quantityInput": {
        "font-size": "18px",
        "padding-top": "17px",
        "padding-bottom": "17px"
      }
    }
  },
  "toggle": {
    "styles": {
      "toggle": {
        "background-color": "#007493",
        ":hover": {
          "background-color": "#006884"
        },
        ":focus": {
          "background-color": "#006884"
        }
      },
      "count": {
        "font-size": "18px"
      }
    }
  },
  "productSet": {
    "styles": {
      "products": {
        "@media (min-width: 601px)": {
          "margin-left": "-20px"
        }
      }
    }
  }
}
          });
        });
      }
    })();
    /*]]>*/
</script>
</html>
</table>
</center></td>
</td></tr></table>


By using DispTEC as the GstDispTEC base, it is possible to perform and run Computer Vision algorithms easily on CPU/GPU, accelerating the performance and delivery time of video analyses. Nevertheless, joining DispTEC with GStreamer, GstDispTEC offers the best of both worlds: the powerful algorithms available in DispTEC and the flexibility of GStreamer to process videos, allowing live-streaming, pipelining and more.
== Overview ==


[[File:Arch diagram.png.png|frame|center|GstDispTEC architecture diagram.]]
GstDispTEC is a GStreamer plug-in which integrates DispTEC <ref>http://palvarado.ietec.org/pmwiki/index.php/Proyectos/DispTEC?userlang=en</ref> library's algorithms, making it possible to incorporate their functionality into GStreamer pipelines, such as motion detection in a video sequence or object tracking, showing where the object is located on a frame, even with non-stationary cameras!
 
''GstDispTEC Arquitecture Description'' illustrates the general architecture of the plug-in and what makes it one of the best solutions for image analysis for GStreamer.
 
= Available Elements =
 
Based on DispTEC library, GstDispTEC currently offers the elements described in the following sections.
 
== Motion detector: dtmotion ==
 
The motion detection element is based on DispTEC's MCD and SCBU algorithms, designed to run on both CPU and GPU. Its primary feature is the motion detection inside a non-stationary capture, although it can also be used on stationary captures. When deciding which of the algorithms to use, consider the following guidelines <ref>https://www.cv-foundation.org/openaccess/content_cvpr_workshops_2013/W03/papers/Yi_Detection_of_Moving_2013_CVPR_paper.pdf</ref>:
* MCD algorithm is faster, especially for smaller resolutions, and the result contains less noise than its counterpart.
* SCBU adjusts better to brightness changes on the captured video but it is more sensible to fast changes in movement. The output is more uniform but contains more noise than MCD.
 
Dtmotion is the designed GStreamer element that integrates the previously described capabilities into a videofilter derived element that can be easily incorporated into a pipeline. It receives any input and generates a new frame built upon a binary mask that exposes the detected motion. In order to adjust the element to the desired application, the following properties can be modified:
{| class="wikitable"
|-
! Name !! Description
|-
| algorithm || Both MCD and SCBU algorithms are available to run on CPU or GPU, depending on the target application.  If not specified, the default MCD algorithm on CPU is selected. The available options are the following:
                          (0): cpu-mcd          - MCD on CPU
                          (1): cpu-scbu        - SCBU on CPU
                          (2): gpu-mcd          - MCD on GPU
                          (3): gpu-scbu        - SCBU On GPU
|-
| block-size || (Only for SCBU) Unsigned Integer. Range: 0 - 4294967295. Default: 4. 
It is an important measure of the algorithm's granularity. The captured image is divided into a number of blocks depending on the specified size. If it is set to be too small, the slightest change between frames may result in motion detection. It will also require greater memory usage, but the processing will be faster and more efficient. Greater value will reduce the amount of noise on the output, and require less memory, but the processing is less efficient.   
|-
| grid-size || (Only for SCBU) Unsigned Integer. Range: 0 - 4294967295 Default: 60 
Grid size determines the distance between the resulting amount of points that will be used to determine how much the background moved. The more points are used, the more precise the measurement. However, the algorithm used is fairly resistant to noise, so it won't make much difference to use a smaller value. A large grid size (fewer points) usually yields good results. 
|-
| max-learning-rate || (Only for SCBU) Unsigned Integer. Range: 0 - 4294967295 Default: 5 
The varying learning rate property is a key feature if trying to detect motion inside of non-static captures, given that it allows adaptation to object movement vs background movement. Increasing the learning rate decreases the sensibility to object motion detection, but improves the rate at which it adapts to background changes. A low maximum value helps in noisy captures due to the fact that the detected moving objects won't be absorbed as background.   
|}
 
== Object tracker: dttracker ==
 
Object tracking has been implemented based on Kernelized Correlation Filters (KCF). Each tracker element is initialized with the coordinates of the object in the current frame, snipping the object from the frame and taking it as a template. Afterwards, the tracker element updates the filter every time that there is an object hit, adapting itself to the possible changes which an object might have during the video, making it robust enough to cope with potential perspective changes.
 
This object tracking technique gives the following features:
 
* Non-stationary camera resilient.
* Tolerates object scaling, rotation and transformation.
* Resilient to changes in colour and light.
 
GstDispTEC object tracker goes even further, making possible to reset the object position at run-time, even if the pipeline is in ''Play'' state. Let's look nearer:
 
* Possibility of tracker cascading.
* Tracker reinitialization at run-time.
* Application integrability.
* Visual feedback of object position overlaid to frames.
* Multi-tracker deployment in the same element (coming soon).
 
 
The tracker element gets some parameters to surround the object to get a template and start tracking. Also, it is possible to enable the visual feedback overlaid on the frame highlighting the current position. These parameters are described below:
 
{| class="wikitable"
|-
! Parameter !! Description
|-
| x || (Unsigned int) Object x coordinate at the top-left corner in pixels. Range: 0 - 4294967295. Default: 0
|-
| y || (Unsigned int) Object y coordinate at the top-left corner in pixels. Range: 0 - 4294967295. Default: 0
|-
| width || (Unsigned int) Object width in pixels. Range: 0 - 4294967295. Default: 100
|-
| height || (Unsigned int) Object height in pixels. Range: 0 - 4294967295. Default: 100
|-
| render-bbox || (Boolean) Render a bounding box which gives a visual representation where the object is currently located. Default: TRUE
|}
 
The tracker element also generates a signal for third-party applications for retrieving the current object position (''coordinates''), whose coordinates are expressed as ''integers'' in the following sequence: x, y, width, and height.
 
= Supported platforms =
 
GstDispTEC currently supports platforms capable to run Open Computer Vision Library (OpenCV). For a better experience, it is recommended to run on a platform with CUDA support for accelerating the ''motion detection'' element. Some of the platforms already tested are:
 
{| class="wikitable"
|-
! Platform !! Operating System 
|-
| Nvidia Jetson TX1 || Ubuntu 16.04 64 bits
|-
| Personal Computer (PC) || Ubuntu 16.04 64 bits
|}
 
Both platforms were running OpenCV 3.3.
 
In case of not having a platform not capable of running CUDA, GstDispTEC will run on CPU.
 
= Building GstDispTEC =
 
== Dependencies ==
 
Before trying to install, make sure of having the following dependencies already installed on the target platform:
 
* OpenCV Core >= 3.3
* OpenCV Contribution Modules >= 3.3
* DispTEC >= 0.2
* CUDA >= 9.0
* GStreamer >= 1.8.1
* Autotools
 
== Installing dependencies ==
 
'''DispTEC'''
 
<syntaxhighlight lang="bash">
sudo apt install cmake
git clone https://github.com/RidgeRun/dispTEC_2017
cmake .
make
sudo make install
</syntaxhighlight>
 
'''OpenCV'''
 
For installing OpenCV, please, follow the steps in the link: [https://docs.opencv.org/3.3.0/d7/d9f/tutorial_linux_install.html OpenCV Installation]
 
'''GStreamer'''
 
<syntaxhighlight lang="bash">
sudo apt-get install \
gstreamer1.0-x \
libgstreamer1.0-dev \
libgstreamer-plugins-base1.0-dev
</syntaxhighlight>
 
'''Autotools'''
 
<syntaxhighlight lang="bash">
sudo apt install \
autotools-dev \
autoconf
</syntaxhighlight>
 
== Installing GstDispTEC ==
 
When you purchase GstDispTEC, you will get a Gitlab repository with the source code inside. You need it to build it in your system and add it to your GStreamer plug-ins catalogue. So, in a temporary path, clone the repository using:
 
<syntaxhighlight lang="bash">
git clone git://git@gitlab.com/RidgeRun/orders/${CUSTOMER_DIRECTORY}/gst-disptec.git
</syntaxhighlight>
 
Where <code>${CUSTOMER_DIRECTORY}</code> contains the name of your customer directory given after the purchase.
 
Afterwards, find the path where GStreamer looks for plug-ins and libraries (<code>LIBDIR</code>). Take as a reference to the following table:
 
{| class="wikitable"
|-
! Platform !! LIBDIR path
|-
| PC 32-bits/x86 || /usr/lib/i386-linux-gnu/
|-
| PC 64-bits/x86 || /usr/lib/x86_64-linux-gnu/
|-
| Nvidia Jetson || /usr/lib/aarch64-linux-gnu/
|}
 
According to the table, define an environment variable which stores the path accordingly to your system. For example, for an x64 PC, it will be:
 
<syntaxhighlight lang="bash">
LIBDIR=/usr/lib/x86_64-linux-gnu/
</syntaxhighlight>
 
Then, do the installation using:
 
<syntaxhighlight lang="bash">
./autogen.sh
./configure --libdir $LIBDIR
make
sudo make install
</syntaxhighlight>
 
== Test installation ==
 
For testing if everything is properly installed, run:
 
<syntaxhighlight lang="bash">
gst-inspect-1.0 disptec
</syntaxhighlight>
 
It should show:
 
<pre>
Plugin Details:
  Name                    disptec
  Description              DispTEC Image processing library plugin
  Filename                /usr/lib/aarch64-linux-gnu/gstreamer-1.0/libgstdisptec.so
  Version                  0.3.0
  License                  Proprietary
  Source module            gst-disptec
  Binary package          RidgeRun
  Origin URL              https://www.ridgerun.com
 
  dttracker: DispTEC tracker element
  dtmotion: DispTEC motion detector
 
  2 features:
  +-- 2 elements
</pre>
 
= Examples =
 
== Object tracking ==
 
'''Example 1: Ball pattern with visual feedback'''
 
<syntaxhighlight lang=bash>
gst-launch-1.0 videotestsrc pattern=ball ! videoconvert ! dttracker name=tracker x=130 y=90 width=50 height=50 ! videoconvert ! xvimagesink
</syntaxhighlight>
 
For this case, the ball always starts at ''x=130'', ''y=90'' and has a width around ''50x50 px''.
 
'''Example 2: Video loaded from a file'''
 
<syntaxhighlight lang=bash>
INITIAL_OBJ_POSITION="x=100 y=100 width=100 height=100"
gst-launch-1.0 filesrc location=$PATH_TO_FILE ! videoconvert ! dttracker name=tracker $INITIAL_OBJ_POSITION ! videoconvert ! xvimagesink
</syntaxhighlight>
 
Set the object initial position into <code>INITIAL_OBJ_POSITION</code>
 
'''Example 3: Capturing from camera'''
 
<syntaxhighlight lang=bash>
INITIAL_OBJ_POSITION="x=100 y=100 width=100 height=100"
gst-launch v4l2src device=$CAPTURE_DEVICE ! $CAPS ! dttracker name=tracker $INITIAL_OBJ_POSITION ! videoconvert ! xvimagesink
</syntaxhighlight>
 
== Motion detector ==
 
'''Example 1: Default settings'''
 
<syntaxhighlight lang=bash>
gst-launch-1.0 filesrc location=$PATH_TO_FILE ! decodebin name=dec ! videoconvert ! dtmotion algorithm=cpu-mcd ! videoconvert ! xvimagesink
</syntaxhighlight>
 
'''Example 2: Setting all properties'''


<syntaxhighlight lang=bash>
With GstDispTEC, it is possible to add several DispTEC-based elements in the same pipeline to strengthen image analysis, connect your own applications to the pipeline for receiving the analysis data, and do much more amazing things. Also, the algorithms can be run either on CPU or GPU, making the most of the computational resources and reducing the processing time.
gst-launch-1.0 filesrc location=$PATH_TO_FILE ! decodebin name=dec ! videoconvert ! dtmotion algorithm=cpu-scbu max-learning-rate=5 block-size=4 grid-size=60 ! videoconvert ! xvimagesink
</syntaxhighlight>


'''Example 3: Capturing from camera'''
== Available Elements ==
The following list contains the DispTEC-based GStreamer plugins currently available:
* [https://developer.ridgerun.com/wiki/index.php?title=GStreamer_DispTEC_Tracker_Plugin Object Tracker Plugin.]
* [https://developer.ridgerun.com/wiki/index.php?title=GStreamer_DispTEC_Motion_Detection_Plugin Motion Detection Plugin]


<syntaxhighlight lang=bash>
{{ContactUs}}
CAPTURE_DEVICE=/dev/video0
CAPS='video/x-raw, format=(string)UYVY, width=(int)640, height=(int)360, framerate=30/1'
gst-launch v4l2src device=$CAPTURE_DEVICE ! $CAPS !  dtmotion algorithm=cpu-mcd ! videoconvert ! xvimagesink
</syntaxhighlight>
= References =

Latest revision as of 14:16, 26 November 2021

Overview

GstDispTEC is a GStreamer plug-in which integrates DispTEC [1] library's algorithms, making it possible to incorporate their functionality into GStreamer pipelines, such as motion detection in a video sequence or object tracking, showing where the object is located on a frame, even with non-stationary cameras!

With GstDispTEC, it is possible to add several DispTEC-based elements in the same pipeline to strengthen image analysis, connect your own applications to the pipeline for receiving the analysis data, and do much more amazing things. Also, the algorithms can be run either on CPU or GPU, making the most of the computational resources and reducing the processing time.

Available Elements

The following list contains the DispTEC-based GStreamer plugins currently available:


RidgeRun Resources

Quick Start Client Engagement Process RidgeRun Blog Homepage
Technical and Sales Support RidgeRun Online Store RidgeRun Videos Contact Us
RidgeRun.ai: Artificial Intelligence | Generative AI | Machine Learning

Contact Us

Visit our Main Website for the RidgeRun Products and Online Store. RidgeRun Engineering information is available at RidgeRun Engineering Services, RidgeRun Professional Services, RidgeRun Subscription Model and Client Engagement Process wiki pages. Please email to support@ridgerun.com for technical questions and contactus@ridgerun.com for other queries. Contact details for sponsoring the RidgeRun GStreamer projects are available in Sponsor Projects page.
  1. ↑ http://palvarado.ietec.org/pmwiki/index.php/Proyectos/DispTEC?userlang=en
Retrieved from "https://developer.ridgerun.com/wiki/index.php?title=GStreamer_DispTEC_Plugin&oldid=37811"