diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..448c788 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +deepstream_parallel_inference_app/tritonserver/models/bodypose2d/1/model.onnx filter=lfs diff=lfs merge=lfs -text +deepstream_parallel_inference_app/tritonserver/models/yolov4/1/yolov4_-1_3_416_416_dynamic.onnx.nms.onnx filter=lfs diff=lfs merge=lfs -text +legacy_apps/deepstream-retail-analytics/files/0001_compressed.h264 filter=lfs diff=lfs merge=lfs -text +legacy_apps/deepstream-retail-analytics/files/basketClassifier.etlt filter=lfs diff=lfs merge=lfs -text +deepstream-tracker-3d-multi-view/archives/datasets.zip filter=lfs diff=lfs merge=lfs -text +deepstream-tracker-3d-multi-view/assets/datasets.zip filter=lfs diff=lfs merge=lfs -text +deepstream-bodypose-3d/streams/bodypose.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index c9d1198..0c9faa1 100644 --- a/.gitignore +++ b/.gitignore @@ -20,12 +20,28 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -sources/**/*.o -sources/**/*.a -sources/**/*.so -sources/**/*.cfg -sources/**/*.weights -sources/**/*.engine -sources/**/deepstream-yolo-app -sources/**/TRT-yolo-app -sources/**/*.jpeg \ No newline at end of file +.vscode/ +tests/ + +**/*.mp4 +**/*.h264 +**/*.caffemodel +**/*.binaryproto +**/*.engine +**/*.weights +**/*.wts +**/*.o +**/*.a +**/*.so + +CaffeMNIST/**/build/ + +yolo/apps/**/deepstream-yolo-app +yolo/apps/**/deepstream-yolo-msgbroker-app +yolo/apps/**/trt-yolo-app +yolo/data/**/*.cfg +yolo/data/**/*.jpeg +yolo/data/**/*.png +yolo/**/build/ +senet/**/build/ +anomaly/**/build/ diff --git a/LICENSE b/LICENSE index f25e38d..971af82 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,16 @@ -MIT License +Apache2.0 License -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. +Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/Makefile.config b/Makefile.config deleted file mode 100644 index 0d3ae84..0000000 --- a/Makefile.config +++ /dev/null @@ -1,27 +0,0 @@ -# MIT License - -# Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -#Update the install directory paths for dependencies below -OPENCV_INSTALL_DIR:= /path/to/opencv-3.4.0 -TENSORRT_INSTALL_DIR:= /path/to/TensorRT-4.0 -DEEPSTREAM_INSTALL_DIR:= /path/to/DeepStream_Release_2.0 \ No newline at end of file diff --git a/README.md b/README.md index c5ea7a4..99667a6 100644 --- a/README.md +++ b/README.md @@ -1,94 +1,50 @@ - -# YOLO Plugin for DeepStream SDK # - -## Installing Pre-requisites: ## - -Download and install DeepStream 2.0 - -Install GStreamer pre-requisites using: - `sudo apt-get install libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev` - -## Setup ## - -In the `Makefile.config` file present in the root directory update install paths for all the dependencies - -### Building NvYolo Plugin ### - -1. Go to the `sources/gst-yoloplugin/yoloplugin_lib/data` directory and add your yolo .cfg and .weights file. - - For yolo v2, - Download the config file from the darknet repo located at `https://github.com/pjreddie/darknet/blob/master/cfg/yolov2.cfg` - Download the weights file by running the command `wget https://pjreddie.com/media/files/yolov2.weights` - - For yolo v3, - Download the config file from the darknet repo located at `https://github.com/pjreddie/darknet/blob/master/cfg/yolov3.cfg` - Download the weights file by running the command `wget https://pjreddie.com/media/files/yolov3.weights` - -4. Set the right macro in the `network_config.h` file to choose a model architecture - -5. [OPTIONAL] Update the paths of the .cfg and .weights file and other network params in `network_config.cpp` file if required. - -7. Add absolute paths of images to be used for calibration in the `calibration_images.txt` file within the `sources/gst-yoloplugin/yoloplugin_lib/data` directory. - -8. Run the following command from `sources/gst-yoloplugin/yoloplugin_lib` to build and install the plugin - `make && sudo make install` - -## Building and running the TRT Yolo App ## - -Go to the `sources/apps/TRT-yolo` directory - -Run the following command to build and install the TRT-yolo-app - `make && sudo make install` - -The TRT Yolo App located at `sources/apps/TRT-yolo` is a sample standalone app, which can be used to run inference on test images. This app does not have any deepstream dependencies and can be built independently. Add a list of absolute paths of images to be used for inference in the `test_images.txt` file located at `yoloplugin_lib/data/` and run `TRT-yolo-app` from the root directory of this repo. Additionally, the detections on test images can be saved by setting `kSAVE_DETECTIONS` config param to `true` in `network_config.cpp` file. The images overlayed with detections will be saved in the `yoloplugin_lib/detections/` directory. - -This app has three command line arguments(optional) that you can pass. One is the batch_size to be used for the TRT engine which is set to 1 by default. The second one is a boolean argument representing if the detections have to be decoded or not which is set to true by default. The last one is an argument to set the seed of random number generators used in the application. It is set to `time(0)` by default. To run the app with the default options run the following command from the root directory of this repo - `TRT-yolo-app` - -To change the batch_size of the TRT engine use the following command - `TRT-yolo-app --batch_size=4` - -## Building and Running DeepStream Yolo App ## - -Go to the `sources/apps/deepstream-yolo` directory - -Run the following command to build and install the deepstream-yolo-app - `make && sudo make install` - -The DeepStream Yolo App located at `sources/apps/deepstream_yolo` is a sample app similar to the Test-1 & Test-2 apps available in the DeepStream SDK. Using the yolo app we build a sample gstreamer pipeline using various components like H264 parser, Decoder, Video Converter, OSD and Yolo plugin to run inference on an elementary h264 video stream. - -Once you have built the deepstream-yolo-app as described above, go to the root directory of this repo and run the command -`deepstream-yolo-app /path/to/sample_video.h264` - -## Running the DeepStream App ## - -Following steps describe how to run the YOLO plugin in the deepstream-app - -1. The section below in the config file corresponds to ds-example(yolo) plugin in deepstream. - The config file is located at `config/deepstream-app_yolo_config.txt`. Make any changes - to this section if required. - - ``` - [ds-example] - enable=1 - processing-width=1280 - processing-height=720 - full-frame=1 - unique-id=15 - gpu-id=0 - ``` - -2. Update path to the video file source in the URI field under `[source0]` group - of the config file - `uri=file://relative/path/to/source/video` - -3. Go to the root folder of this repo and run - `deepstream-app -c config/deepstream-app_yolo_config.txt` - -### Note ### - -1. If you are using the plugin with deepstream-app (located at `/usr/bin/deepstream-app`), register the yolo plugin as dsexample. To do so, replace line 671 in `gstyoloplugin.cpp` with `return gst_element_register(plugin, "dsexample", GST_RANK_PRIMARY, GST_TYPE_YOLOPLUGIN);` - - This registers the plugin with the name `dsexample` so that the deepstream-app can pick it up and add to it's pipeline. Now go to `sources/gst-yoloplugin/` and run `make && sudo make install` to build and install the plugin. - -2. Tegra users who are currently using Deepstream 1.5, please use the standalone TRT app as your starting point and incorporate that inference pipeline in your inference plugin. \ No newline at end of file +# This repository has ceased updates, please refer to https://github.com/NVIDIA/deepstream for latest DeepStream reference applications. + +# Reference Apps using DeepStream 9.0 + +This repository contains the reference applications for video analytics tasks using TensorRT and DeepSTream SDK 9.0. + +## Getting Started ## +We currently provide three different reference applications: + +Preferably clone this project in +`/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/` + +To clone the project in the above directory, sudo permission may be required. + +For further details, please see each project's README. + +### Anomaly Detection : [README](anomaly/README.md) ### + The project contains auxiliary dsdirection plugin to show the capability of DeepstreamSDK in anomaly detection. + ![sample anomaly output](anomaly/.opticalflow.png) +### Runtime Source Addition Deletion: [README](runtime_source_add_delete/README.md) ### + The project demonstrates addition and deletion of video sources in a live Deepstream pipeline. +### MaskTracker: [README](deepstream-masktracker/README.md) ### + This sample app demonstrates DeepStream MaskTracker for multi-object tracking and segmentation using SAM2. + ![sample MaskTracker output](deepstream-masktracker/figures/.retail_osd.gif) +### Single-View 3D Tracking: [README](deepstream-tracker-3d/README.md) ### + The sample app demonstrates single-view 3D tracking with DeepStream multi-object tracking to reconstruct 3D human model in world coordinates under occlusion. + ![sample 3D tracking output](deepstream-tracker-3d/figures/.retail_viz.png) +### Multi-View 3D Tracking: [README](deepstream-tracker-3d-multi-view) ### + The samples demonstrate multi-view 3D tracking in DeepStream, a distributed, real-time framework designed for large-scale, calibrated camera networks. + + sample mulit-view 3D tracking output sample BEV output from multi-view 3D tracking +### Parallel Multiple Models Inferencing: [README](deepstream_parallel_inference_app/README.md) ### + The project demonstrate how to implement multiple models inferencing in parallel with DeepStream APIs. +### Bodypose 3D Model Inferencing: [README](deepstream-bodypose-3d/README.md) ### + The sample demonstrate how to customize the multiple input layers model preprocessing and the customization of the bodypose 3D model postprocessing. + ![Bodypose 3D sample output](deepstream-bodypose-3d/sources/.screenshot.png) +### Video Buffers sharing between pipelines through IPC: [README](deepstream-ipc-test-sr/README.md) ### + This sample demonstrates how to share video buffers over IPC and how to change output video buffers. +### Multiple Dynamic Sources with Single Decoder:[README](deepstream-dynamicsrcbin-test/README) ### + The sample demonstrates the usage of nvdsdynamicsrcbin in DeepStream pipeline.It helps to construct the application of multiple dynamic sources with a single video decoder to adapt to the high decoder throughput scenarios. +### Custom Video Tiling Config: [README](deepstream-custom-tile-config/README.md) ### + This sample demonstrates the usage of "custom-tile-config" of nvmultistreamtiler to customize the tiling positions and sizes of multiple videos in batch. +### DeepStream VLLM Plugin: [README](deepstream-vllm-plugin/README.md) ### + A GStreamer plugin for NVIDIA DeepStream that integrates Vision-Language Models (VLM) using VLLM for real-time video understanding and analysis. + +## Pyservicemaker Sample apps +The apps in pyservicemaker_sample_apps are additional samples demonstrating usage of the Python API for DeepStream Service Maker, either by flow API or by pipeline API. See the [Python Service Maker documentation](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_service_maker_python.html) for details. + +## Legacy DeepStream Reference Samples +Some old samples are not supported any more for different reasons. The legacy samples are moved to the legacy folder /legacy_apps. diff --git a/anomaly/.dsdirection_pipeline.png b/anomaly/.dsdirection_pipeline.png new file mode 100755 index 0000000..d4b8434 Binary files /dev/null and b/anomaly/.dsdirection_pipeline.png differ diff --git a/anomaly/.opticalflow.png b/anomaly/.opticalflow.png new file mode 100755 index 0000000..8ee9c87 Binary files /dev/null and b/anomaly/.opticalflow.png differ diff --git a/anomaly/README.md b/anomaly/README.md new file mode 100755 index 0000000..413a55a --- /dev/null +++ b/anomaly/README.md @@ -0,0 +1,80 @@ +# ANOMALY DETECTION REFERENCE APP USING DEEPSTREAMSDK 9.0 + +## Introduction +The project contains anomaly detection application and auxiliary plug-ins to show the +capability of Deepstream SDK. + +## Prequisites: +DeepStream SDK installed which is available at http://developer.nvidia.com/deepstream-sdk +Please follow instructions in the `apps/sample_apps/deepstream-app/README` on how +to install the prequisites for Deepstream SDK apps. + +## Getting Started + +- Preferably clone the app in + `/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/` + +- Edit the `dsanomaly_pgie_config.txt` or `dsanomaly_pgie_nvinferserver_config.txt` according to the location of the models to be used + + +## Compilation Steps for dsdirection plugin +``` + $ cd plugins/gst-dsdirection/ + $ sudo make && sudo make install +``` + +1. Test direction calculation on one video input, on dGPU, run following commands +``` +cd /opt/nvidia/deepstream/deepstream/ +gst-launch-1.0 filesrc location = samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_0 \ +nvstreammux name=m batch-size=1 width=1920 height=1080 ! nvinfer config-file-path= samples/configs/deepstream-app/config_infer_primary.txt \ +! nvof ! tee name=t ! queue ! nvofvisual ! nvmultistreamtiler width=1920 height=1080 ! nveglglessink t. ! queue ! dsdirection ! \ +nvmultistreamtiler width=1920 height=1080 ! nvvideoconvert ! nvdsosd ! nveglglessink +``` +2. Test direction calculation on one video input, on Jetson, run following commands +``` +cd /opt/nvidia/deepstream/deepstream/ +gst-launch-1.0 filesrc location = samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_0 \ +nvstreammux name=m batch-size=1 width=1280 height=720 ! nvinfer config-file-path= samples/configs/deepstream-app/config_infer_primary.txt \ +! nvof ! tee name=t ! queue ! nvofvisual ! nvmultistreamtiler width=1920 height=1080 ! nv3dsink sync=0 t. ! queue ! dsdirection ! \ +nvmultistreamtiler width=1920 height=1080 ! nvvideoconvert ! nvdsosd ! nv3dsink sync=0 +``` + +3. Test direction calculation using optical flow on two video inputs on dGPU, run following commands +``` +cd /opt/nvidia/deepstream/deepstream/ +gst-launch-1.0 filesrc location = samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_0 \ +nvstreammux name=m batch-size=2 width=1920 height=1080 ! nvinfer config-file-path= samples/configs/deepstream-app/config_infer_primary.txt ! \ +nvof ! tee name=t ! queue ! nvofvisual ! nvmultistreamtiler width=1920 height=540 ! nveglglessink t. ! queue ! dsdirection ! \ +nvmultistreamtiler width=1920 height=540 ! nvvideoconvert ! nvdsosd ! nveglglessink filesrc location = samples/streams/sample_1080p_h264.mp4 ! \ +qtdemux ! h264parse ! nvv4l2decoder ! m.sink_1 --gst-debug=3 + +``` +Anomaly detection app pipeline: +![DS Anomaly Detection Pipeline](.dsdirection_pipeline.png) + +## Compilation Steps and Execution: +``` + $ cd sources/apps/sample_apps/deepstream_reference_apps/anomaly/ + $ cd apps/deepstream-anomaly-detection-test/ + $ Set CUDA_VER in the MakeFile as per platform. + For x86, CUDA_VER=13.1 + For jetson, CUDA_VER=13.0 + $ sudo make + + $ ./deepstream-anomaly-detection-app [uri2] ... [uriN] + Ex.: ./deepstream-anomaly-detection-app file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 + +Use option "-t inferserver" to select nvinferserver as the inference plugin + $ ./deepstream-anomaly-detection-app -t inferserver [uri2] ... [uriN] + Ex.: ./deepstream-anomaly-detection-app -t inferserver file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 +``` + The result should be like below: + ![DS Anomaly Detection Screenshot](.opticalflow.png) + +## NOTE: +- Minimum supported resolution: DGPU - 160 x 64, Jetson - 256 x 96 +- Due to an issue in nvofvisual plugin, when using nvofvisual along with nvof + plugin, the width of input to nvof should be multiple of 32 on DGPU and multiple + of 256 on Jetson. This will be fixed in the nvofvisual plugin in the next DeepStream + release. diff --git a/anomaly/apps/deepstream-anomaly-detection-test/Makefile b/anomaly/apps/deepstream-anomaly-detection-test/Makefile new file mode 100755 index 0000000..43bb407 --- /dev/null +++ b/anomaly/apps/deepstream-anomaly-detection-test/Makefile @@ -0,0 +1,61 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +APP:= deepstream-anomaly-detection-app + +TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -) + +DS_SDK_ROOT:=/opt/nvidia/deepstream/deepstream + +LIB_INSTALL_DIR?=$(DS_SDK_ROOT)/lib/ + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= -I$(DS_SDK_ROOT)/sources/includes \ + -I /usr/local/cuda-$(CUDA_VER)/include + +CFLAGS+= `pkg-config --cflags $(PKGS)` + +LIBS:= `pkg-config --libs $(PKGS)` + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvdsgst_helper -lm \ + -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart \ + -lcuda -Wl,-rpath,$(LIB_INSTALL_DIR) + +all: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +clean: + rm -rf $(OBJS) $(APP) + + diff --git a/anomaly/apps/deepstream-anomaly-detection-test/deepstream_anomaly_detection_test.c b/anomaly/apps/deepstream-anomaly-detection-test/deepstream_anomaly_detection_test.c new file mode 100755 index 0000000..158ebef --- /dev/null +++ b/anomaly/apps/deepstream-anomaly-detection-test/deepstream_anomaly_detection_test.c @@ -0,0 +1,502 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "gstnvdsmeta.h" +#include "gst-nvmessage.h" + +/* The muxer output resolution must be set if the input streams will be of + * different resolution. The muxer will scale all the input frames to this + * resolution. */ +#define MUXER_OUTPUT_WIDTH 1280 +#define MUXER_OUTPUT_HEIGHT 720 + +/* Muxer batch formation timeout, for e.g. 33 millisec. Should ideally be set + * based on the fastest source's framerate. */ +#define MUXER_BATCH_TIMEOUT_USEC 33000 + +#define TILED_OUTPUT_WIDTH_INFER 1280 +#define TILED_OUTPUT_HEIGHT_INFER 720 + +#define TILED_OUTPUT_WIDTH_OF 640 +#define TILED_OUTPUT_HEIGHT_OF 360 + +#define NVINFER_PLUGIN "nvinfer" +#define NVINFERSERVER_PLUGIN "nvinferserver" + +#define PGIE_CONFIG_FILE "dsanomaly_pgie_config.txt" +#define PGIE_NVINFERSERVER_CONFIG_FILE "dsanomaly_pgie_nvinferserver_config.txt" + +/* NVIDIA Decoder source pad memory feature. This feature signifies that source + * pads having this capability will push GstBuffers containing NvBufSurface. */ +#define GST_CAPS_FEATURES_NVMM "memory:NVMM" + +static gboolean +bus_call (GstBus * bus, GstMessage * msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + g_print ("End of stream\n"); + g_main_loop_quit (loop); + break; + case GST_MESSAGE_WARNING: + { + gchar *debug; + GError *error; + gst_message_parse_warning (msg, &error, &debug); + g_printerr ("WARNING from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + g_free (debug); + g_printerr ("Warning: %s\n", error->message); + g_error_free (error); + break; + } + case GST_MESSAGE_ERROR: + { + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + case GST_MESSAGE_ELEMENT: + { + if (gst_nvmessage_is_stream_eos (msg)) { + guint stream_id; + if (gst_nvmessage_parse_stream_eos (msg, &stream_id)) { + g_print ("Got EOS from stream %d\n", stream_id); + } + } + break; + } + default: + break; + } + return TRUE; +} + +static void +cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data) +{ + g_print ("In cb_newpad\n"); + GstCaps *caps = gst_pad_get_current_caps (decoder_src_pad); + const GstStructure *str = gst_caps_get_structure (caps, 0); + const gchar *name = gst_structure_get_name (str); + GstElement *source_bin = (GstElement *) data; + GstCapsFeatures *features = gst_caps_get_features (caps, 0); + + /* Need to check if the pad created by the decodebin is for video and not + * audio. */ + if (!strncmp (name, "video", 5)) { + /* Link the decodebin pad only if decodebin has picked nvidia + * decoder plugin nvv4l2decoder. We do this by checking if the pad caps contain + * NVMM memory features. */ + if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) { + /* Get the source bin ghost pad */ + GstPad *bin_ghost_pad = gst_element_get_static_pad (source_bin, "src"); + if (!gst_ghost_pad_set_target (GST_GHOST_PAD (bin_ghost_pad), + decoder_src_pad)) { + g_printerr ("Failed to link decoder src pad to source bin ghost pad\n"); + } + gst_object_unref (bin_ghost_pad); + } else { + g_printerr ("Error: Decodebin did not pick nvidia decoder plugin.\n"); + } + } +} + +static void +decodebin_child_added (GstChildProxy * child_proxy, GObject * object, + gchar * name, gpointer user_data) +{ + g_print ("Decodebin child added: %s\n", name); + if (g_strrstr (name, "decodebin") == name) { + g_signal_connect (G_OBJECT (object), "child-added", + G_CALLBACK (decodebin_child_added), user_data); + } +} + +static GstElement * +create_source_bin (guint index, gchar * uri) +{ + GstElement *bin = NULL, *uri_decode_bin = NULL; + gchar bin_name[16] = { }; + + g_snprintf (bin_name, 15, "source-bin-%02d", index); + /* Create a source GstBin to abstract this bin's content from the rest of the + * pipeline */ + bin = gst_bin_new (bin_name); + + /* Source element for reading from the uri. + * We will use decodebin and let it figure out the container format of the + * stream and the codec and plug the appropriate demux and decode plugins. */ + uri_decode_bin = gst_element_factory_make ("uridecodebin", "uri-decode-bin"); + + if (!bin || !uri_decode_bin) { + g_printerr ("One element in source bin could not be created.\n"); + return NULL; + } + + /* We set the input uri to the source element */ + g_object_set (G_OBJECT (uri_decode_bin), "uri", uri, NULL); + + /* Connect to the "pad-added" signal of the decodebin which generates a + * callback once a new pad for raw data has beed created by the decodebin */ + g_signal_connect (G_OBJECT (uri_decode_bin), "pad-added", + G_CALLBACK (cb_newpad), bin); + g_signal_connect (G_OBJECT (uri_decode_bin), "child-added", + G_CALLBACK (decodebin_child_added), bin); + + gst_bin_add (GST_BIN (bin), uri_decode_bin); + + /* We need to create a ghost pad for the source bin which will act as a proxy + * for the video decoder src pad. The ghost pad will not have a target right + * now. Once the decode bin creates the video decoder and generates the + * cb_newpad callback, we will set the ghost pad target to the video decoder + * src pad. */ + if (!gst_element_add_pad (bin, gst_ghost_pad_new_no_target ("src", + GST_PAD_SRC))) { + g_printerr ("Failed to add ghost pad in source bin\n"); + return NULL; + } + + return bin; +} + +static void +usage(const char *bin) +{ + g_printerr ("Usage: %s [uri2] ... [uriN]\n", bin); + g_printerr ("For nvinferserver, Usage: %s -t inferserver [uri2] ... [uriN]\n", bin); +} + +int +main (int argc, char *argv[]) +{ + GMainLoop *loop = NULL; + GstElement *pipeline = NULL, *streammux = NULL, *streammux_queue = NULL, + *sink_of = NULL, *pgie_queue = NULL, *dsdirection_queue = NULL, + *nvvidconv_queue = NULL, *nvosd_queue = NULL, *tiler_infer_queue = NULL, + *tiler_of = NULL, *nvof = NULL, *nvofvisual = NULL, *dsdirection = NULL, + *of_queue = NULL, *ofvisual_queue = NULL, *sink_infer = NULL, + *tiler_infer = NULL, *pgie = NULL, *nvvidconv = NULL, + *nvosd = NULL, *tee = NULL, *of_branch_queue = NULL, *infer_branch_queue = + NULL; + + GstBus *bus = NULL; + guint bus_watch_id; + guint i, num_sources; + guint tiler_rows, tiler_columns; + guint pgie_batch_size; + gboolean is_nvinfer_server = FALSE; + + GstPad *tee_of_pad, *tee_infer_pad; + GstPad *queue_of_pad, *queue_infer_pad; + + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + /* Check input arguments */ + if (argc < 2) { + usage(argv[0]); + return -1; + } + + if (argc >=2 && !strcmp("-t", argv[1])) { + if (!strcmp("inferserver", argv[2])) { + is_nvinfer_server = TRUE; + } else { + usage(argv[0]); + return -1; + } + g_print ("Using nvinferserver as the inference plugin\n"); + } + + if (is_nvinfer_server) { + num_sources = argc - 3; + } else { + num_sources = argc - 1; + } + + /* Standard GStreamer initialization */ + gst_init (&argc, &argv); + loop = g_main_loop_new (NULL, FALSE); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + pipeline = gst_pipeline_new ("anomaly-detection-pipeline"); + + /* Create nvstreammux instance to form batches from one or more sources. */ + streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); + streammux_queue = gst_element_factory_make ("queue", "streammux-queue"); + + if (!pipeline || !streammux || !streammux_queue) { + g_printerr ("(Line=%d) One element could not be created. Exiting.\n", + __LINE__); + return -1; + } + gst_bin_add (GST_BIN (pipeline), streammux); + + for (i = 0; i < num_sources; i++) { + GstPad *sinkpad, *srcpad; + GstElement *source_bin; + gchar pad_name[16] = { }; + if (is_nvinfer_server) { + source_bin = create_source_bin (i, argv[i + 3]); + } else { + source_bin = create_source_bin (i, argv[i + 1]); + } + + if (!source_bin) { + g_printerr ("Failed to create source bin. Exiting.\n"); + return -1; + } + + gst_bin_add (GST_BIN (pipeline), source_bin); + + g_snprintf (pad_name, 15, "sink_%u", i); + sinkpad = gst_element_get_request_pad (streammux, pad_name); + if (!sinkpad) { + g_printerr ("Streammux request sink pad failed. Exiting.\n"); + return -1; + } + + srcpad = gst_element_get_static_pad (source_bin, "src"); + if (!srcpad) { + g_printerr ("Failed to get src pad of source bin. Exiting.\n"); + return -1; + } + + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Failed to link source bin to stream muxer. Exiting.\n"); + return -1; + } + + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + } + + /* Create a tee for two sinks. */ + tee = gst_element_factory_make ("tee", "tee"); + + /* Use nvinfer/nvinferserver to infer on batched frame. */ + pgie = gst_element_factory_make ( + is_nvinfer_server ? NVINFERSERVER_PLUGIN : NVINFER_PLUGIN, + "primary-nvinference-engine"); + pgie_queue = gst_element_factory_make ("queue", "nvinfer-queue"); + + /* For Optical Flow output */ + tiler_of = gst_element_factory_make ("nvmultistreamtiler", "nvtiler-of"); + /* Use nvtiler to composite the batched frames into a 2D tiled array based + * on the source of the frames. */ + tiler_infer = + gst_element_factory_make ("nvmultistreamtiler", "nvtiler-infer"); + tiler_infer_queue = + gst_element_factory_make ("queue", "nvtiler-infer-queue"); + + /* create nv optical flow element */ + nvof = gst_element_factory_make ("nvof", "nvopticalflow"); + + /* create nv ds direction element */ + dsdirection = gst_element_factory_make ("dsdirection", "dsdirection"); + dsdirection_queue = gst_element_factory_make ("queue", "dsdirection-queue"); + + /* create nv optical flow visualisation element */ + nvofvisual = gst_element_factory_make ("nvofvisual", "nvopticalflowvisual"); + + /* create queue element */ + of_queue = gst_element_factory_make ("queue", "q_after_of"); + + /* create queue element */ + ofvisual_queue = gst_element_factory_make ("queue", "q_after_ofvisual"); + + /* create queue element */ + of_branch_queue = gst_element_factory_make ("queue", "q_of"); + + /* create queue element */ + infer_branch_queue = gst_element_factory_make ("queue", "q_infer"); + + /* Use convertor to convert from NV12 to RGBA as required by nvosd */ + nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); + nvvidconv_queue = gst_element_factory_make ("queue", "nvvideoconvert-queue"); + + /* Create OSD to draw on the converted RGBA buffer */ + nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); + nvosd_queue = gst_element_factory_make ("queue", "nvdsosd-queue"); + + /* Finally render the osd output */ + if(prop.integrated) { + sink_of = gst_element_factory_make ("nv3dsink", "nv3dsink-of"); + sink_infer = + gst_element_factory_make ("nv3dsink", "nv3dsink-infer"); + + } else { +#ifdef __aarch64__ + sink_of = gst_element_factory_make ("nv3dsink", "nv3dsink-of"); + sink_infer = + gst_element_factory_make ("nv3dsink", "nv3dsink-infer"); +#else + sink_of = gst_element_factory_make ("nveglglessink", "nvelgglessink-of"); + sink_infer = + gst_element_factory_make ("nveglglessink", "nvelgglessink-infer"); +#endif + } + + if (!tee) { + g_printerr ("Tee could not be created. Exiting.\n"); + return -1; + } + + if (!nvof || !dsdirection || !nvofvisual || !tiler_of || !sink_of) { + g_printerr ("One OF element could not be created. Exiting.\n"); + return -1; + } + + if (!pgie || !tiler_infer || !nvvidconv || !nvosd || !sink_infer) { + g_printerr ("One Infer element could not be created. Exiting.\n"); + return -1; + } + + if (!pgie_queue || !tiler_infer_queue || !nvvidconv_queue || !nvosd_queue) { + g_printerr ("One Queue element could not be created. Exiting.\n"); + return -1; + } + + /* We set the sync value of both sink elements */ + g_object_set (G_OBJECT (sink_of), "sync", 1, NULL); + g_object_set (G_OBJECT (sink_infer), "sync", 1, NULL); + + g_object_set (G_OBJECT (streammux), "sync-inputs", TRUE, NULL); + g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", + MUXER_OUTPUT_HEIGHT, "batch-size", num_sources, + "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); + + /* Configure the nvinfer/nvinferserver element using the config file. */ + if (is_nvinfer_server) { + g_object_set (G_OBJECT (pgie), "config-file-path", PGIE_NVINFERSERVER_CONFIG_FILE, NULL); + } else { + g_object_set (G_OBJECT (pgie), "config-file-path", PGIE_CONFIG_FILE, NULL); + } + + /* Override the batch-size set in the config file with the number of sources. */ + g_object_get (G_OBJECT (pgie), "batch-size", &pgie_batch_size, NULL); + if (pgie_batch_size != num_sources) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + pgie_batch_size, num_sources); + g_object_set (G_OBJECT (pgie), "batch-size", num_sources, NULL); + } + + tiler_rows = (guint) sqrt (num_sources); + tiler_columns = (guint) ceil (1.0 * num_sources / tiler_rows); + /* we set the tiler properties here */ + g_object_set (G_OBJECT (tiler_of), "rows", tiler_rows, "columns", + tiler_columns, "width", TILED_OUTPUT_WIDTH_OF, "height", + TILED_OUTPUT_HEIGHT_OF, NULL); + g_object_set (G_OBJECT (tiler_infer), "rows", tiler_rows, "columns", + tiler_columns, "width", TILED_OUTPUT_WIDTH_INFER, "height", + TILED_OUTPUT_HEIGHT_INFER, NULL); + + /* We set the sink properties here */ + g_object_set (G_OBJECT (sink_of), "window-x", 0, "window-y", 0, NULL); + g_object_set (G_OBJECT (sink_infer), "window-x", TILED_OUTPUT_WIDTH_OF, + "window-y", TILED_OUTPUT_HEIGHT_OF, NULL); + + /* we add a message handler */ + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); + gst_object_unref (bus); + + /* Set up the pipeline */ + /* we add all elements into the pipeline */ + gst_bin_add_many (GST_BIN (pipeline), streammux_queue, pgie, pgie_queue, + nvof, of_queue, dsdirection, dsdirection_queue, tee, + of_branch_queue, nvofvisual, ofvisual_queue, tiler_of, sink_of, + infer_branch_queue, tiler_infer, tiler_infer_queue, nvvidconv, + nvvidconv_queue, nvosd, nvosd_queue, sink_infer, NULL); + + + if ((!gst_element_link_many (streammux, streammux_queue, pgie, pgie_queue, + nvof, of_queue, dsdirection, dsdirection_queue, tee, NULL)) + || (!gst_element_link_many (of_branch_queue, nvofvisual, ofvisual_queue, + tiler_of, NULL)) + || (!gst_element_link_many (infer_branch_queue, tiler_infer, + tiler_infer_queue, nvvidconv, nvvidconv_queue, nvosd, + nvosd_queue, NULL)) || + (!gst_element_link_many (tiler_of, sink_of, NULL)) || + (!gst_element_link_many (nvosd_queue, sink_infer, NULL))) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + + + /* Manually link the Tee, which has "Request" pads */ + tee_of_pad = gst_element_get_request_pad (tee, "src_%u"); + g_print ("Obtained request pad %s for OF branch.\n", + gst_pad_get_name (tee_of_pad)); + queue_of_pad = gst_element_get_static_pad (of_branch_queue, "sink"); + tee_infer_pad = gst_element_get_request_pad (tee, "src_%u"); + g_print ("Obtained request pad %s for infer branch.\n", + gst_pad_get_name (tee_infer_pad)); + queue_infer_pad = gst_element_get_static_pad (infer_branch_queue, "sink"); + if (gst_pad_link (tee_of_pad, queue_of_pad) != GST_PAD_LINK_OK || + gst_pad_link (tee_infer_pad, queue_infer_pad) != GST_PAD_LINK_OK) { + g_printerr ("Tee could not be linked.\n"); + gst_object_unref (pipeline); + return -1; + } + gst_object_unref (queue_of_pad); + gst_object_unref (queue_infer_pad); + + /* Set the pipeline to "playing" state */ + g_print ("Now playing...\n"); + + GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS (GST_BIN (pipeline), + GST_DEBUG_GRAPH_SHOW_ALL, "nvof_test_playing"); + + gst_element_set_state (pipeline, GST_STATE_PLAYING); + + /* Wait till pipeline encounters an error or EOS */ + g_print ("Running...\n"); + g_main_loop_run (loop); + + /* Out of the main loop, clean up nicely */ + g_print ("Returned, stopping playback\n"); + gst_element_set_state (pipeline, GST_STATE_NULL); + g_print ("Deleting pipeline\n"); + gst_object_unref (GST_OBJECT (pipeline)); + g_source_remove (bus_watch_id); + g_main_loop_unref (loop); + gst_deinit (); + return 0; +} diff --git a/anomaly/apps/deepstream-anomaly-detection-test/dsanomaly_pgie_config.txt b/anomaly/apps/deepstream-anomaly-detection-test/dsanomaly_pgie_config.txt new file mode 100755 index 0000000..260e26b --- /dev/null +++ b/anomaly/apps/deepstream-anomaly-detection-test/dsanomaly_pgie_config.txt @@ -0,0 +1,71 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# enable-dbscan(Default=false), interval(Primary mode only, Default=0) +# custom-lib-path +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.00392156862745098 +onnx-file=../../../../../../../samples/models/Primary_Detector/resnet18_trafficcamnet_pruned.onnx +model-engine-file=../../../../../../../samples/models/Primary_Detector/resnet18_trafficcamnet_pruned.onnx_b1_gpu0_fp16.engine +labelfile-path=../../../../../../../samples/models/Primary_Detector/labels.txt +int8-calib-file=../../../../../../../samples/models/Primary_Detector/cal_trt.bin +batch-size=1 +process-mode=1 +model-color-format=0 +network-mode=2 +num-detected-classes=4 +interval=0 +gie-unique-id=1 +cluster-mode=2 + +[class-attrs-all] +pre-cluster-threshold=0.2 +topk=20 +nms-iou-threshold=0.5 diff --git a/anomaly/apps/deepstream-anomaly-detection-test/dsanomaly_pgie_nvinferserver_config.txt b/anomaly/apps/deepstream-anomaly-detection-test/dsanomaly_pgie_nvinferserver_config.txt new file mode 100755 index 0000000..c7fc3f1 --- /dev/null +++ b/anomaly/apps/deepstream-anomaly-detection-test/dsanomaly_pgie_nvinferserver_config.txt @@ -0,0 +1,74 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 30 + backend { + inputs: [ { + name: "input_1:0" + }] + outputs: [ + {name: "output_cov/Sigmoid:0"}, + {name: "output_bbox/BiasAdd:0"} + ] + triton { + model_name: "Primary_Detector" + version: -1 + model_repo { + root: "../../../../../../../samples/triton_model_repo" + strict_model_config: true + } + } + } + preprocess { + network_format: MEDIA_FORMAT_NONE + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + channel_offsets: [0, 0, 0] + } + } + postprocess { + labelfile_path: "../../../../../../../samples/models/Primary_Detector/labels.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/anomaly/plugins/gst-dsdirection/Makefile b/anomaly/plugins/gst-dsdirection/Makefile new file mode 100755 index 0000000..9145ab6 --- /dev/null +++ b/anomaly/plugins/gst-dsdirection/Makefile @@ -0,0 +1,63 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# + +CXX:= g++ +SRCS:= gstdsdirection.cpp +INCS:= $(wildcard *.h) +LIB:=libnvdsgst_dsdirection.so + +DS_SDK_ROOT:=/opt/nvidia/deepstream/deepstream + +DEP:=dsdirection_lib/libdsdirection.a +DEP_FILES:=$(wildcard dsdirection_lib/dsdirection_lib.* ) +DEP_FILES-=$(DEP) + +CFLAGS+= -fPIC -DDS_VERSION=\"6.0.0\" \ + -I $(DS_SDK_ROOT)/sources/includes + +GST_INSTALL_DIR?=$(DS_SDK_ROOT)/lib/gst-plugins/ +LIB_INSTALL_DIR?=$(DS_SDK_ROOT)/lib/ + +LIBS := -shared -Wl,-no-undefined \ + -L dsdirection_lib -ldsdirection \ + -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta \ + -Wl,-rpath,$(LIB_INSTALL_DIR) + +OBJS:= $(SRCS:.cpp=.o) + +PKGS:= gstreamer-1.0 gstreamer-base-1.0 +CFLAGS+=$(shell pkg-config --cflags $(PKGS)) +LIBS+=$(shell pkg-config --libs $(PKGS)) + +all: $(LIB) + +%.o: %.cpp $(INCS) Makefile + @echo $(CFLAGS) + $(CXX) -c -o $@ $(CFLAGS) $< + +$(LIB): $(OBJS) $(DEP) Makefile + @echo $(CFLAGS) + $(CXX) -o $@ $(OBJS) $(LIBS) $(DEP) + +$(DEP): $(DEP_FILES) + $(MAKE) -C dsdirection_lib/ + +install: $(LIB) + cp -rv $(LIB) $(GST_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(LIB) diff --git a/anomaly/plugins/gst-dsdirection/dsdirection_lib/Makefile b/anomaly/plugins/gst-dsdirection/dsdirection_lib/Makefile new file mode 100755 index 0000000..f524c41 --- /dev/null +++ b/anomaly/plugins/gst-dsdirection/dsdirection_lib/Makefile @@ -0,0 +1,26 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# + +PKGS:= gstreamer-1.0 gstreamer-base-1.0 + +DS_SDK_ROOT:=/opt/nvidia/deepstream/deepstream +CFLAGS+=-I $(DS_SDK_ROOT)/sources/includes +CFLAGS+=$(shell pkg-config --cflags $(PKGS)) + +all: + gcc -ggdb $(CFLAGS) -c -o dsdirection_lib.o -fPIC dsdirection_lib.cpp + ar rcs libdsdirection.a dsdirection_lib.o diff --git a/anomaly/plugins/gst-dsdirection/dsdirection_lib/dsdirection_lib.cpp b/anomaly/plugins/gst-dsdirection/dsdirection_lib/dsdirection_lib.cpp new file mode 100755 index 0000000..f7fc198 --- /dev/null +++ b/anomaly/plugins/gst-dsdirection/dsdirection_lib/dsdirection_lib.cpp @@ -0,0 +1,112 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "dsdirection_lib.h" +#include +#include +#include +using namespace std; + +#define PI 3.141592 +#define PI_IN_DEGREES 180 +//NVOF gives quarter-pixel output using OFSDK +#define FACTOR_QPEL 4.0 + + +struct DsDirectionLabel +{ + //Start of interval in degrees + float start; + //End of interval in degrees + float end; + //Direction in UTF-8 Format + const char *dirName; +}; + +static const DsDirectionLabel label[8] = { + {157.5, -157.5, "\u21D0"}, //left + {-67.5, -22.5, "\u21D8"}, //bottom-right + {-112.5, -67.5, "\u21D3"}, //bottom + {-157.5, -112.5, "\u21D9"}, //bottom-left + {-22.5, 22.5, "\u21D2"}, //right + {112.5, 157.5, "\u21D6"}, //top left + {67.5, 112.5, "\u21D1"}, //top + {22.5, 67.5, "\u21D7"} //top-right +}; + +DsDirectionOutput * +DsDirectionProcess (NvOFFlowVector * in_flow, int flow_cols, int flow_rows, + int flow_bsize, NvOSD_RectParams * rect_param) +{ + DsDirectionOutput *out = + (DsDirectionOutput *) calloc (1, sizeof (DsDirectionOutput)); + //Mean for the whole object Bounding Box + float x_mean = 0, y_mean = 0, max_radius = 0; + //Frequency or no of pixels in the bounding box + int freq_pix = 0; + //Sum of Optical Flow value of pixels + float x_mean_sum = 0, y_mean_sum = 0; + + //Get the motion inside the bbox. Sum it in x and y direction. + for (unsigned int j = rect_param->top; + j <= rect_param->top + rect_param->height; j++) { + unsigned int block_j = j / flow_bsize; + for (unsigned int i = rect_param->left; + i <= rect_param->left + rect_param->width; i++) { + // To get the mapping to the optical flow data (in_flow) + unsigned int block_i = i / flow_bsize; + unsigned int pos = block_j * flow_cols + block_i; + + //IF condition removes zero-motion pixels. thus getting a better estimate + if ((in_flow[pos].flowx != 0) || in_flow[pos].flowy != 0) { + x_mean_sum += (in_flow[pos].flowx) / FACTOR_QPEL; + y_mean_sum += (in_flow[pos].flowy) / FACTOR_QPEL; + freq_pix++; + } + } + } + + //Final Mean + x_mean = ((float) x_mean_sum) / freq_pix; + y_mean = ((float) y_mean_sum) / freq_pix; + out->object.flowx = x_mean; + out->object.flowy = -y_mean; + max_radius = sqrt (x_mean * x_mean + y_mean * y_mean); + + int dir = -1; + //Assigning the direction based on optical flow data + if (max_radius > 2) { + //dir initialized to 0 to take care of special case for left direction. + dir = 0; + float angle = (atan2 (-y_mean, x_mean) * PI_IN_DEGREES) / PI; + for (int i = 0; i <= 7; i++) { + if (angle > label[i].start && angle <= label[i].end) { + dir = i; + break; + } + } + } + + if (dir >= 0 && dir < 8) { + snprintf (out->object.direction, 128, "%s", label[dir].dirName); + } else { + //Blank output when threshold is not crossed. + snprintf (out->object.direction, 128, "%s", ""); + } + + return out; +} diff --git a/anomaly/plugins/gst-dsdirection/dsdirection_lib/dsdirection_lib.h b/anomaly/plugins/gst-dsdirection/dsdirection_lib/dsdirection_lib.h new file mode 100755 index 0000000..8b69127 --- /dev/null +++ b/anomaly/plugins/gst-dsdirection/dsdirection_lib/dsdirection_lib.h @@ -0,0 +1,54 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __DSDIRECTION_LIB__ +#define __DSDIRECTION_LIB__ +#include "nvll_osd_struct.h" +#include "nvds_opticalflow_meta.h" + +#define MAX_LABEL_SIZE 128 +#ifdef __cplusplus +extern "C" +{ +#endif + + +// Detected/Labelled object structure, stores bounding box info along with label +typedef struct +{ + float flowx; + float flowy; + char direction[MAX_LABEL_SIZE]; +} DsDirectionObject; + +// Output data returned after processing +typedef struct +{ + DsDirectionObject object; +} DsDirectionOutput; + +// Dequeue processed output +DsDirectionOutput *DsDirectionProcess (NvOFFlowVector * in_flow, + int flow_cols, int flow_rows, int flow_bsize, + NvOSD_RectParams * rect_param); + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/anomaly/plugins/gst-dsdirection/gstdsdirection.cpp b/anomaly/plugins/gst-dsdirection/gstdsdirection.cpp new file mode 100755 index 0000000..7ee4985 --- /dev/null +++ b/anomaly/plugins/gst-dsdirection/gstdsdirection.cpp @@ -0,0 +1,339 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "gstdsdirection.h" +#include "nvds_opticalflow_meta.h" +GST_DEBUG_CATEGORY_STATIC (gst_dsdirection_debug); +#define GST_CAT_DEFAULT gst_dsdirection_debug + +static GQuark _dsmeta_quark = 0; + +/* Enum to identify properties */ +enum +{ + PROP_0, + PROP_UNIQUE_ID +}; +// Block size used in NVOF application +#define NVOF_BLK_SIZE 4 +/*set the user metadata type*/ +#define NVDS_DIRECTION_USER_META (nvds_get_user_meta_type(((gchar *)"NVIDIA.NVDSDIRECTION.DIR_META"))) +/* Default values for properties */ +#define DEFAULT_UNIQUE_ID 15 + +/* By default NVIDIA Hardware allocated memory flows through the pipeline. We + * will be processing on this type of memory only. */ +#define GST_CAPS_FEATURE_MEMORY_NVMM "memory:NVMM" +static GstStaticPadTemplate gst_dsdirection_sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_NVMM, + "{ NV12, RGBA, I420 }"))); + +static GstStaticPadTemplate gst_dsdirection_src_template = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_NVMM, + "{ NV12, RGBA, I420 }"))); + +/* Define our element type. Standard GObject/GStreamer boilerplate stuff */ +#define gst_dsdirection_parent_class parent_class +G_DEFINE_TYPE (GstDsDirection, gst_dsdirection, GST_TYPE_BASE_TRANSFORM); + +static void gst_dsdirection_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_dsdirection_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static GstFlowReturn gst_dsdirection_transform_ip (GstBaseTransform * + btrans, GstBuffer * inbuf); + + +static void attach_metadata_object (GstDsDirection * dsdirection, + NvDsObjectMeta * obj_meta, DsDirectionOutput * output); + +/* Install properties, set sink and src pad capabilities, override the required + * functions of the base class, These are common to all instances of the + * element. + */ +static void +gst_dsdirection_class_init (GstDsDirectionClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstBaseTransformClass *gstbasetransform_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + gstbasetransform_class = (GstBaseTransformClass *) klass; + + /* Overide base class functions */ + gobject_class->set_property = + GST_DEBUG_FUNCPTR (gst_dsdirection_set_property); + gobject_class->get_property = + GST_DEBUG_FUNCPTR (gst_dsdirection_get_property); + + gstbasetransform_class->transform_ip = + GST_DEBUG_FUNCPTR (gst_dsdirection_transform_ip); + + /* Install properties */ + g_object_class_install_property (gobject_class, PROP_UNIQUE_ID, + g_param_spec_uint ("unique-id", + "Unique ID", + "Unique ID for the element. Can be used to identify output of the" + " element", 0, G_MAXUINT, DEFAULT_UNIQUE_ID, (GParamFlags) + (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + /* Set sink and src pad capabilities */ + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&gst_dsdirection_src_template)); + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&gst_dsdirection_sink_template)); + + /* Set metadata describing the element */ + gst_element_class_set_details_simple (gstelement_class, + "DsDirection plugin", + "DsDirection Plugin", + "Estimate direction in which object is moving", + "NVIDIA Corporation. Post on Deepstream for Tesla forum for any queries " + "@ https://devtalk.nvidia.com/default/board/209/"); +} + +static void +gst_dsdirection_init (GstDsDirection * dsdirection) +{ + GstBaseTransform *btrans = GST_BASE_TRANSFORM (dsdirection); + + /* We will not be generating a new buffer. Just adding / updating + * metadata. */ + gst_base_transform_set_in_place (GST_BASE_TRANSFORM (btrans), TRUE); + /* We do not want to change the input caps. Set to passthrough. transform_ip + * is still called. */ + gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (btrans), TRUE); + + /* Initialize all property variables to default values */ + dsdirection->unique_id = DEFAULT_UNIQUE_ID; + /* This quark is required to identify NvDsMeta when iterating through + * the buffer metadatas */ + if (!_dsmeta_quark) + _dsmeta_quark = g_quark_from_static_string (NVDS_META_STRING); +} + +/* Function called when a property of the element is set. Standard boilerplate. +*/ +static void +gst_dsdirection_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstDsDirection *dsdirection = GST_DSDIRECTION (object); + switch (prop_id) { + case PROP_UNIQUE_ID: + dsdirection->unique_id = g_value_get_uint (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +/* Function called when a property of the element is requested. Standard + * boilerplate. + */ +static void +gst_dsdirection_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstDsDirection *dsdirection = GST_DSDIRECTION (object); + switch (prop_id) { + case PROP_UNIQUE_ID: + g_value_set_uint (value, dsdirection->unique_id); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +/** + * Called when element recieves an input buffer from upstream element. + */ +static GstFlowReturn +gst_dsdirection_transform_ip (GstBaseTransform * btrans, GstBuffer * inbuf) +{ + GstDsDirection *dsdirection = GST_DSDIRECTION (btrans); + DsDirectionOutput *output; + NvDsBatchMeta *batch_meta = NULL; + NvDsFrameMeta *frame_meta = NULL; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsObjectMeta *obj_meta = NULL; + dsdirection->frame_num++; + + GST_DEBUG_OBJECT (dsdirection, + "Processing Frame %lu", dsdirection->frame_num); + + batch_meta = gst_buffer_get_nvds_batch_meta (inbuf); + if (batch_meta == nullptr) { + GST_ELEMENT_ERROR (dsdirection, STREAM, FAILED, + ("NvDsBatchMeta not found for input buffer."), (NULL)); + return GST_FLOW_ERROR; + } + //Iterating through frames in batched meta from diff. sources + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + + frame_meta = (NvDsFrameMeta *) (l_frame->data); + NvDsFrameMetaList *fmeta_list = NULL; + //Iterating through each frame of the batched meta + for (fmeta_list = frame_meta->frame_user_meta_list; fmeta_list != NULL; + fmeta_list = fmeta_list->next) { + NvDsUserMeta *of_user_meta = NULL; + //previous meta + optical flow meta + of_user_meta = (NvDsUserMeta *) fmeta_list->data; + + if (of_user_meta + && of_user_meta->base_meta.meta_type == NVDS_OPTICAL_FLOW_META) { + //optical flow meta for each frame + NvDsOpticalFlowMeta *ofmeta = + (NvDsOpticalFlowMeta *) (of_user_meta->user_meta_data); + if (ofmeta) { + //Iterating through each object in the frame + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) { + obj_meta = (NvDsObjectMeta *) (l_obj->data); + + //processing the meta data for direction detection + output = + DsDirectionProcess ((NvOFFlowVector *) ofmeta->data, + ofmeta->cols, ofmeta->rows, NVOF_BLK_SIZE, + &obj_meta->rect_params); + // Attach direction to the object + attach_metadata_object (dsdirection, obj_meta, output); + + } + } + } + } + } + return GST_FLOW_OK; +} + +/* copy function set by user. "data" holds a pointer to NvDsUserMeta*/ +static gpointer +copy_ds_direction_meta (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + DsDirectionOutput *src_user_metadata + = (DsDirectionOutput *) user_meta->user_meta_data; + DsDirectionOutput *dst_user_metadata + = (DsDirectionOutput *) calloc (1, sizeof (DsDirectionOutput)); + memcpy (dst_user_metadata, src_user_metadata, sizeof (DsDirectionOutput)); + return (gpointer) dst_user_metadata; +} + +/* release function set by user. "data" holds a pointer to NvDsUserMeta*/ +static void +release_ds_direction_meta (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + if (user_meta->user_meta_data) { + free (user_meta->user_meta_data); + user_meta->user_meta_data = NULL; + } +} + +/** + * Only update string label in an existing object metadata. No bounding boxes. + * We assume only one label per object is generated + */ +static void +attach_metadata_object (GstDsDirection * dsdirection, NvDsObjectMeta * obj_meta, + DsDirectionOutput * output) +{ + NvDsBatchMeta *batch_meta = obj_meta->base_meta.batch_meta; + + // Attach - DsDirection MetaData + NvDsUserMeta *user_meta = nvds_acquire_user_meta_from_pool (batch_meta); + NvDsMetaType user_meta_type = NVDS_DIRECTION_USER_META; + + user_meta->user_meta_data = output; + user_meta->base_meta.meta_type = user_meta_type; + user_meta->base_meta.copy_func = copy_ds_direction_meta; + user_meta->base_meta.release_func = release_ds_direction_meta; + + nvds_add_user_meta_to_obj (obj_meta, user_meta); + + nvds_acquire_meta_lock (batch_meta); + NvOSD_TextParams & text_params = obj_meta->text_params; + NvOSD_RectParams & rect_params = obj_meta->rect_params; + + /* Below code to display the result */ + // Set black background for the text + // display_text required heap allocated memory + if (text_params.display_text) { + gchar *conc_string = g_strconcat (text_params.display_text, " ", + output->object.direction, NULL); + g_free (text_params.display_text); + text_params.display_text = conc_string; + text_params.font_params.font_size = 12; + } else { + // Display text above the left top corner of the object + text_params.x_offset = rect_params.left; + text_params.y_offset = rect_params.top - 10; + text_params.display_text = g_strdup (output->object.direction); + // Font face, size and color + text_params.font_params.font_name = (char *) "Serif"; + text_params.font_params.font_size = 15; + text_params.font_params.font_color = (NvOSD_ColorParams) { + 1, 1, 1, 1}; + // Set black background for the text + text_params.set_bg_clr = 1; + text_params.text_bg_clr = (NvOSD_ColorParams) { + 0, 0, 0, 1}; + } + nvds_release_meta_lock (batch_meta); +} + +/** + * Boiler plate for registering a plugin and an element. + */ +static gboolean +dsdirection_plugin_init (GstPlugin * plugin) +{ + GST_DEBUG_CATEGORY_INIT (gst_dsdirection_debug, "dsdirection", 0, + "dsdirection plugin"); + + return gst_element_register (plugin, "dsdirection", GST_RANK_PRIMARY, + GST_TYPE_DSDIRECTION); +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + nvdsgst_dsdirection, + DESCRIPTION, dsdirection_plugin_init, DS_VERSION, LICENSE, BINARY_PACKAGE, + URL) diff --git a/anomaly/plugins/gst-dsdirection/gstdsdirection.h b/anomaly/plugins/gst-dsdirection/gstdsdirection.h new file mode 100755 index 0000000..a17feda --- /dev/null +++ b/anomaly/plugins/gst-dsdirection/gstdsdirection.h @@ -0,0 +1,81 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __GST_DSDIRECTION_H__ +#define __GST_DSDIRECTION_H__ + +#include +#include + +#include "gstnvdsmeta.h" +#include "dsdirection_lib/dsdirection_lib.h" + +/* Package and library details required for plugin_init */ +#define PACKAGE "dsdirection" +#define VERSION "1.0" +#define LICENSE "Proprietary" +#define DESCRIPTION "NVIDIA dsdirection plugin for integration with DeepStream" +#define BINARY_PACKAGE "NVIDIA DeepStream 3rdparty IP integration dsdirection plugin" +#define URL "http://nvidia.com/" + + +G_BEGIN_DECLS +/* Standard boilerplate stuff */ +typedef struct _GstDsDirection GstDsDirection; +typedef struct _GstDsDirectionClass GstDsDirectionClass; + +/* Standard boilerplate stuff */ +#define GST_TYPE_DSDIRECTION (gst_dsdirection_get_type()) +#define GST_DSDIRECTION(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_DSDIRECTION,GstDsDirection)) +#define GST_DSDIRECTION_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_DSDIRECTION,GstDsDirectionClass)) +#define GST_DSDIRECTION_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj), GST_TYPE_DSDIRECTION, GstDsDirectionClass)) +#define GST_IS_DSDIRECTION(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_DSDIRECTION)) +#define GST_IS_DSDIRECTION_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_DSDIRECTION)) +#define GST_DSDIRECTION_CAST(obj) ((GstDsDirection *)(obj)) + +struct _GstDsDirection +{ + GstBaseTransform base_trans; + + // Unique ID of the element. The labels generated by the element will be + // updated at index `unique_id` of attr_info array in NvDsObjectParams. + guint unique_id; + + // Frame number of the current input buffer + guint64 frame_num; + + // Input video info (resolution, color format, framerate, etc) + GstVideoInfo video_info; + + // Amount of objects processed in single call to algorithm + guint batch_size; + + // GPU ID on which we expect to execute the task + guint gpu_id; + +}; + +// Boiler plate stuff +struct _GstDsDirectionClass +{ + GstBaseTransformClass parent_class; +}; + +GType gst_dsdirection_get_type (void); + +G_END_DECLS +#endif /* __GST_DSDIRECTION_H__ */ diff --git a/config/deepstream-app_yolo_config.txt b/config/deepstream-app_yolo_config.txt deleted file mode 100644 index bec4b93..0000000 --- a/config/deepstream-app_yolo_config.txt +++ /dev/null @@ -1,54 +0,0 @@ -[application] -enable-perf-measurement=1 -perf-measurement-interval-sec=5 -flow-original-resolution=1 -#gie-kitti-output-dir=streamscl - -[tiled-display] -enable=1 -rows=1 -columns=1 -width=1280 -height=720 -gpu-id=0 - -[source0] -enable=1 -#Type - 1=CameraV4L2 2=URI 3=MultiURI -type=3 -uri=file://relative/path/to/source/video -num-sources=1 -gpu-id=0 - -[sink0] -enable=1 -#Type - 1=FakeSink 2=EglSink 3=File -type=2 -sync=1 -source-id=0 -gpu-id=0 - -[osd] -enable=1 -gpu-id=0 -osd-mode=1 -border-width=1 -text-size=15 -text-color=1;1;1;1; -text-bg-color=0.3;0.3;0.3;1 -font=Arial -show-clock=0 -clock-x-offset=800 -clock-y-offset=820 -clock-text-size=12 -clock-color=1;0;0;0 - -[tests] -file-loop=0 - -[ds-example] -enable=1 -processing-width=1280 -processing-height=720 -full-frame=1 -unique-id=15 \ No newline at end of file diff --git a/deepstream-3d-sensor-fusion/README.md b/deepstream-3d-sensor-fusion/README.md new file mode 100755 index 0000000..0ab38ce --- /dev/null +++ b/deepstream-3d-sensor-fusion/README.md @@ -0,0 +1,123 @@ +## **What's DS3D Multi-modal sensor fusion** + +The ``deepstream-3d-lidar-sensor-fusion`` sample application showcases multi-modal sensor fusion pipelines for LiDAR and camera data using the DS3D framework. This appliation with DS3D framework could setup different LiDAR/RADAR/Camera sensor fusion models, late fusion inference pipelines with several key features. + +- Camera processing pipeline leveraging DeepStream’s generic 2D video pipeline with batchMeta. +- Custom ds3d::dataloader for LiDAR capture with pre-processing options. +- Custom ds3d::databridge converts DeepStream NvBufSurface and GstNvDsPreProcessBatchMeta data into shaped based tensor data s3d::Frame2DGuard and ds3d::FrameGuard formats, and embeds key-value pairs within ds3d::datamap. +- ds3d::mixer for efficient merging of camera, LiDAR and any sensor data into ds3d::datamap. +- ds3d::datatfiler followed by libnvds_tritoninferfilter.so for multi-modal ds3d::datamap inference and custom pre/post-processing. +- ds3d::datasink with ds3d_gles_ensemble_render for 3D detection result visualization with a multi-view display. + +### Requirements: + + - [Deepstream SDK 7.0+](https://developer.nvidia.com/deepstream-sdk) + +### Sample and source code location: + +```bash +cd /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-3d-lidar-sensor-fusion/ +``` +### Further details: + +Refer to the DeepStream documentation for a detailed explanation of this sample application: +[DS3D Multimodal Sensor Fusion](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_3D_MultiModal_Lidar_Sensor_Fusion.html) + + +## **About the ds3d-bevfusion NuScenes dataset (data/nuscene.tar.gz)** + +This dataset is a subset of the original NuScenes dataset containing LiDAR and camera samples for the multi-modal sensor fusion demo. It's licensed for non-commercial use. See [The Terms of Use](https://www.nuscenes.org/terms-of-use) + +### Downloading the dataset: + +Before running the BEVFusion demo, users need to download ``data/nuscene.tar.gz``. After clone this repo, Users can achieve this using git-lfs commands: + +```bash +git lfs install +git lfs fetch +git lfs checkout +``` + +### Extracting the dataset: + +```bash +mkdir /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-3d-lidar-sensor-fusion/data/ +cp data/nuscene.tar.gz /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-3d-lidar-sensor-fusion/data/ +cd /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-3d-lidar-sensor-fusion/data/ +tar -pxvf nuscene.tar.gz +``` + +Instructions for setting up the demo follow in the next section. + + +## **What is DS3D BEVFusion pipeline of Sensor Fusion** + +The pipeline efficiently processes data from 6 cameras and 1 LiDAR, leveraging a pre-trained PyTorch [BEVFusion model](https://github.com/mit-han-lab/bevfusion). This model is optimized for NVIDIA GPUs using TensorRT and CUDA, as showcased by the [CUDA-BEVFusion](https://github.com/NVIDIA-AI-IOT/Lidar_AI_Solution/tree/master/CUDA-BEVFusion) project. Enhancing integration, a multi-modal inference module based on [PyTriton](https://github.com/triton-inference-server/pytriton) simplifies the incorporation of the Python BEVFusion model. The pipeline seamlessly employs ``ds3d::datatfiler`` for triton inference through gRPC. Ultimately, users can visualize the results by examining the ``ds3d::datamap`` with 6 camera views. The pipeline also projects LiDAR data and 3D bounding boxes into each view. Furthermore, the same LiDAR data is thoughtfully visualized in both a top view and a front view for enhanced comprehension. + +### **Quick setup** + +Follow the instructions in DeepStream documentation [DS3D Multimodal Sensor Fusion](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_3D_MultiModal_Lidar_Sensor_Fusion.html) + +### **Detailed setup** + +For a more detailed explanation of the BEVFusion pipeline with NuScenes calibration and other datasets, refer to [DS3D Multimodal Lidar Camera BEVFusion](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_3D_MultiModal_Lidar_Camera_BEVFusion.html) + +## **Instructions to collect NuScenes dataset for this specific demo scene** + +This dataset ``data/nuscene.tar.gz`` could be re-collected through the following scripts from official NuScenes dataset. + +```bash +cd /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-3d-lidar-sensor-fusion +python3 python/triton_lmm/helper/nuscene_data_setup.py --data_dir=dataset/nuscene \ +--ds3d_fusion_workspace=/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-3d-lidar-sensor-fusion --print_calibration +``` + +The script ``nuscene_data_setup.py`` is typically included within the DeepStream SDK installation (version 7.0 or later). You can search it by: + +```bash +find /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-3d-lidar-sensor-fusion/python -name nuscene_data_setup.py +``` + +## **License of NuScenes dataset (data/nuscene.tar.gz)** + +This tiny dataset ``data/nuscene.tar.gz`` is collected from the official nuScenes website, is licensed under +```bash +Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Public License (“CC BY-NC-SA 4.0”). +``` +[term-of-use](https://www.nuscenes.org/terms-of-use) + +## **Citation** + +- BEVFusion model is originally from [github bevfusion](https://github.com/mit-han-lab/bevfusion) + +```bibtex +@inproceedings{liu2022bevfusion, + title={BEVFusion: Multi-Task Multi-Sensor Fusion with Unified Bird's-Eye View Representation}, + author={Liu, Zhijian and Tang, Haotian and Amini, Alexander and Yang, Xingyu and Mao, Huizi and Rus, Daniela and Han, Song}, + booktitle={IEEE International Conference on Robotics and Automation (ICRA)}, + year={2023} +} +``` + +- nuScenes Dataset + +```bibtex +@article{nuscenes2019, + title={nuScenes: A multimodal dataset for autonomous driving}, + author={Holger Caesar and Varun Bankiti and Alex H. Lang and Sourabh Vora and + Venice Erin Liong and Qiang Xu and Anush Krishnan and Yu Pan and + Giancarlo Baldan and Oscar Beijbom}, + journal={arXiv preprint arXiv:1903.11027}, + year={2019} +} +``` + +```bibtex +@article{fong2021panoptic, + title={Panoptic nuScenes: A Large-Scale Benchmark for LiDAR Panoptic Segmentation and Tracking}, + author={Fong, Whye Kit and Mohan, Rohit and Hurtado, Juana Valeria and Zhou, Lubing and Caesar, Holger and + Beijbom, Oscar and Valada, Abhinav}, + journal={arXiv preprint arXiv:2109.03805}, + year={2021} +} +``` diff --git a/deepstream-3d-sensor-fusion/data/.gitattributes b/deepstream-3d-sensor-fusion/data/.gitattributes new file mode 100755 index 0000000..a7fd07a --- /dev/null +++ b/deepstream-3d-sensor-fusion/data/.gitattributes @@ -0,0 +1 @@ +nuscene.tar.gz filter=lfs diff=lfs merge=lfs -text diff --git a/deepstream-3d-sensor-fusion/data/nuscene.tar.gz b/deepstream-3d-sensor-fusion/data/nuscene.tar.gz new file mode 100755 index 0000000..efa6ad9 --- /dev/null +++ b/deepstream-3d-sensor-fusion/data/nuscene.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1aad08a23b7e90f3ca625258526321aaa92032e22c7af6712819311cb8bca344 +size 52016896 diff --git a/deepstream-bodypose-3d/LICENSE b/deepstream-bodypose-3d/LICENSE new file mode 100755 index 0000000..07a5f42 --- /dev/null +++ b/deepstream-bodypose-3d/LICENSE @@ -0,0 +1,16 @@ +SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + diff --git a/deepstream-bodypose-3d/README.md b/deepstream-bodypose-3d/README.md new file mode 100755 index 0000000..ae5c329 --- /dev/null +++ b/deepstream-bodypose-3d/README.md @@ -0,0 +1,133 @@ +# 3d-bodypose-deepstream + +## Introduction +The project contains 3D Body Pose application built using Deepstream SDK. + +This application is built for [KAMA: 3D Keypoint Aware Body Mesh Articulation](https://arxiv.org/abs/2104.13502). +![sample pose output](./sources/.screenshot.png) +## Prerequisites: +DeepStream SDK 9.0 installed which is available at http://developer.nvidia.com/deepstream-sdk +Please follow instructions in the `/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-app/README` on how +to install the prequisites for building Deepstream SDK apps. + +The pretrained TAO models [PeopleNet](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet) and [BodyPose3DNet](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/bodypose3dnet) from NGC. + +## Installation +Follow https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_Quickstart.html to setup the DeepStream SDK + +1. Preferably clone the app in + `/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/` +and define project home as `export BODYPOSE3D_HOME=/3d-bodypose-deepstream`. + +2. Install Eigen development packages +``` + sudo apt install libeigen3-dev + cd /usr/include + sudo ln -sf eigen3/Eigen Eigen +``` + +3. For Deepstream SDK version older than 6.2, copy and build custom `NvDsEventMsgMeta` into Deepstream SDK installation path. Copy and build custom `NvDsEventMsgMeta` into Deepstream SDK installation path. +The custom `NvDsEventMsgMeta` structure handles pose3d and pose25d meta data. +```bash +# Copy deepstream sources +cp $BODYPOSE3D_HOME/sources/deepstream-sdk/eventmsg_payload.cpp /opt/nvidia/deepstream/deepstream/sources/libs/nvmsgconv/deepstream_schema +# Build new nvmsgconv library for custom Product metadata +cd /opt/nvidia/deepstream/deepstream/sources/libs/nvmsgconv +make; make install +``` +Please note that this step is not necessary for Deepstream SDK version 6.2 or newer. + +## Build the applications +```bash +# Build custom nvinfer parser of BodyPose3DNet +cd $BODYPOSE3D_HOME/sources/nvdsinfer_custom_impl_BodyPose3DNet +make +# Build deepstream-pose-estimation-app +cd $BODYPOSE3D_HOME/sources +make +``` +If the above steps are successful, `deepstream-pose-estimation-app` shall be built in the same directory. Under `$BODYPOSE3D_HOME/sources/nvdsinfer_custom_impl_BodyPose3DNet`, `libnvdsinfer_custom_impl_BodyPose3DNet.so` should be present as well. + +## Run the applications +### `deepstream-pose-estimation-app` +Download the pretrained TAO models with the download script +``` +cd $BODYPOSE3D_HOME +bash ./download_models.sh +``` + +The command line options of this application are listed below: +```bash +$ ./deepstream-pose-estimation-app -h +Usage: + deepstream-pose-estimation-app [OPTION?] Deepstream BodyPose3DNet App + +Help Options: + -h, --help Show help options + --help-all Show all help options + --help-gst Show GStreamer Options + +Application Options: + -v, --version Print DeepStreamSDK version. + --version-all Print DeepStreamSDK and dependencies version. + --input [Required] Input video address in URI format by starting with "rtsp://" or "file://". + --output Output video address. Either "rtsp://" or a file path is acceptable. If the value is "rtsp://", then the result video is published at "rtsp://localhost:8554/ds-test". + --save-pose The file path to save both the pose25d and the recovered pose3d in JSON format. + --conn-str Connection string for Gst-nvmsgbroker, e.g. ;;. + --publish-pose Specify the type of pose to publish. Acceptable value is either "pose3d" or "pose25d". If not specified, both "pose3d" and "pose25d" are published to the message broker. + --tracker Specify the NvDCF tracker mode. The acceptable value is either "accuracy" or "perf". The default value is "accuracy". + --fps Print FPS in the format of current_fps (averaged_fps). + --width Input video width in pixels. The default value is 1280. + --height Input video height in pixels. The default value is 720. + --focal Camera focal length in millimeters. The default value is 800.79041. +``` + +Here are examples running this application: +1. Below command processes an input video in URI format and renders the overlaid pose estimation in a window. +```bash +$ ./deepstream-pose-estimation-app --input file://$BODYPOSE3D_HOME/streams/bodypose.mp4 +``` +Please provide the absolute path to the source video file. + +2. When the data source is a video file, below command saves the output video with the skeleton overlay to `$BODYPOSE3D_HOME/streams/bodypose_3dbp.mp4` and save the skeleton's keypoints to `$BODYPOSE3D_HOME/streams/bodypose_3dbp.json`. +```bash +$ ./deepstream-pose-estimation-app --input file://$BODYPOSE3D_HOME/streams/bodypose.mp4 --output $BODYPOSE3D_HOME/streams/bodypose_3dbp.mp4 --focal 800.0 --width 1280 --height 720 --fps --save-pose $BODYPOSE3D_HOME/streams/bodypose_3dbp.json +``` +`bodypose_3dbp.json` contains the predicted 34 keypoints in both `pose25d` and `pose3d` space: +```bash +[{ + "num_frames_in_batch": 1, + "batches": [{ + "batch_id": 0, + "frame_num": 3, + "ntp_timestamp": 1639431716322229000, + "num_obj_meta": 6, + "objects": [{ + "object_id": 3, + "pose25d": [707.645203, 338.592499, -0.000448, 0.867188, ...], + "pose3d": [297.649933, -94.196518, 3520.129883, 0.867188, ...] + },{ + ... + }] + }] +}, { +``` +`pose25d` contains `34x4` floats. A four-item group represents a keypoint's `[x, y, zRel, conf]` +values. `x` and `y` are the keypoint's position in the image coordinate; `zRel` is the relative +depth value from the skeleton's root keypoint, i.e. pelvis. `x, y, zRel` values are in millimeters. +`conf` is the confidence value of the prediction. + +`pose3d` also contains `34x4` floats. A four-item group represents a keypoint's `[x, y, z, conf]` +values. `x`, `y`, `z` are the keypoint's 3D position in the world coordinate whose origin is the +camera. `x, y, z` values are in millimeters. `conf` is the confidence value of the prediction. + +3. When the data source is an RTSP stream and the result is published to RTSP stream `rtsp://localhost:8554/ds-test`, +```bash +$ ./deepstream-pose-estimation-app --input rtsp://:/ --output rtsp:// +``` + +4. In order to publish both pose3D and pose25D metadata to a message broker, please do +```bash +$ ./deepstream-pose-estimation-app --input file://$BODYPOSE3D_HOME/streams/bodypose.mp4 --conn-str "localhost;9092;test" +``` +where `\"localhost;9092;test\"` is the connection string to the message broker `localhost`, port number `9092`, and topic name `test`. Please apply double quotes around the connection string since `;` is a reserved character in shell. diff --git a/deepstream-bodypose-3d/configs/config_infer_primary_peoplenet.txt b/deepstream-bodypose-3d/configs/config_infer_primary_peoplenet.txt new file mode 100755 index 0000000..b1b8975 --- /dev/null +++ b/deepstream-bodypose-3d/configs/config_infer_primary_peoplenet.txt @@ -0,0 +1,48 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +onnx-file=../models/peoplenet/resnet34_peoplenet_int8.onnx +int8-calib-file=../models/peoplenet/resnet34_peoplenet_int8.txt +labelfile-path=../models/peoplenet/labels.txt +model-engine-file=../models/peoplenet/resnet34_peoplenet_int8.onnx_b1_gpu0_int8.engine +infer-dims=3;544;960 +output-blob-names=output_bbox/BiasAdd:0;output_cov/Sigmoid:0 +batch-size=1 +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=1 +num-detected-classes=3 +cluster-mode=2 +interval=0 +gie-unique-id=1 +maintain-aspect-ratio=0 + +## Use the config params below for NMS clustering mode +[class-attrs-all] +topk=8 +nms-iou-threshold=0.5 +pre-cluster-threshold=0.2 + +## Per class configurations +[class-attrs-0] +topk=20 +nms-iou-threshold=0.5 +pre-cluster-threshold=0.4 diff --git a/deepstream-bodypose-3d/configs/config_infer_secondary_bodypose3dnet.txt b/deepstream-bodypose-3d/configs/config_infer_secondary_bodypose3dnet.txt new file mode 100755 index 0000000..0f04b40 --- /dev/null +++ b/deepstream-bodypose-3d/configs/config_infer_secondary_bodypose3dnet.txt @@ -0,0 +1,50 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=0.00392156 +model-engine-file=../models/bodypose3dnet/bodypose3dnet_accuracy.onnx_b8_gpu0_fp16.engine +onnx-file=../models/bodypose3dnet/bodypose3dnet_accuracy.onnx +#model-engine-file=../models/bodypose3dnet/bodypose3dnet_performance.onnx_b8_gpu0_fp16.engine +#onnx-file=../models/bodypose3dnet/bodypose3dnet_performance.onnx +infer-dims=3;256;192 +batch-size=8 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +## 0=Detection 1=Classifier 2=Segmentation 100=other +network-type=100 +num-detected-classes=1 +interval=0 +gie-unique-id=2 +output-blob-names=pose2d;pose2d_org_img;pose25d;pose3d +classifier-threshold=0.7 +operate-on-class-ids=0 +## Integer 0:NCHW 1:NHWC +network-input-order=0 +# Enable tensor metadata output +output-tensor-meta=1 +## 1-Primary 2-Secondary +process-mode=2 +## 0=RGB 1=BGR 2=GRAY +model-color-format=1 +maintain-aspect-ratio=0 +symmetric-padding=0 +scaling-filter=1 +custom-lib-path=../sources/nvdsinfer_custom_impl_BodyPose3DNet/libnvdsinfer_custom_impl_BodyPose3DNet.so + + diff --git a/deepstream-bodypose-3d/download_models.sh b/deepstream-bodypose-3d/download_models.sh new file mode 100755 index 0000000..bf5d681 --- /dev/null +++ b/deepstream-bodypose-3d/download_models.sh @@ -0,0 +1,36 @@ +#!/bin/sh +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +echo "===================================================================" +echo "begin downloading PeopleNet model " +echo "===================================================================" +mkdir -p ./models/peoplenet +cd ./models/peoplenet +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=resnet34_peoplenet_int8.onnx' -O resnet34_peoplenet_int8.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=resnet34_peoplenet_int8.txt' -O resnet34_peoplenet_int8.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=labels.txt' -O labels.txt + +echo "===================================================================" +echo "begin downloading BodyPose3DNet model " +echo "===================================================================" +cd - +mkdir -p ./models/bodypose3dnet +cd ./models/bodypose3dnet +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/bodypose3dnet/deployable_accuracy_onnx_1.0/files?redirect=true&path=bodypose3dnet_accuracy.onnx' -O bodypose3dnet_accuracy.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/bodypose3dnet/deployable_performance_onnx_v1.0/files?redirect=true&path=bodypose3dnet_performance.onnx' -O bodypose3dnet_performance.onnx + diff --git a/deepstream-bodypose-3d/sources/.screenshot.png b/deepstream-bodypose-3d/sources/.screenshot.png new file mode 100755 index 0000000..b291037 Binary files /dev/null and b/deepstream-bodypose-3d/sources/.screenshot.png differ diff --git a/deepstream-bodypose-3d/sources/Makefile b/deepstream-bodypose-3d/sources/Makefile new file mode 100755 index 0000000..70d24e8 --- /dev/null +++ b/deepstream-bodypose-3d/sources/Makefile @@ -0,0 +1,86 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +CXX=g++ -std=c++14 + +APP:= deepstream-pose-estimation-app + +TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -) + +CUDA_HOME:= /usr/local/cuda-$(CUDA_VER) +DEEPSTREAM_HOME:= /opt/nvidia/deepstream/deepstream + +LIB_INSTALL_DIR?=$(DEEPSTREAM_HOME)/lib/ +APP_INSTALL_DIR?=$(DEEPSTREAM_HOME)/bin/ + +ifeq ($(TARGET_DEVICE),aarch64) + CFLAGS:= -DPLATFORM_TEGRA +endif + +SRCS:= deepstream_pose_estimation_app.cpp + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 gstreamer-video-1.0 x11 json-glib-1.0 + +OBJS:= $(patsubst %.c,%.o, $(patsubst %.cpp,%.o, $(SRCS))) + +CFLAGS+= \ + -I$(CUDA_HOME)/include \ + -I$(DEEPSTREAM_HOME)/sources/includes \ + -I$(DEEPSTREAM_HOME)/sources/apps/apps-common/includes \ + -I$(DEEPSTREAM_HOME)/sources/apps/sample_apps/deepstream-app \ + -I../eigen \ + -DDS_VERSION_MINOR=2 -DDS_VERSION_MAJOR=6 + +LIBS+= \ + -L$(CUDA_HOME)/lib64 -lcudart -lcuda \ + -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvds_utils -lgstapp-1.0 \ + -lpthread -lm -ldl -Wl,-rpath,$(LIB_INSTALL_DIR) + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) + +LIBS+= $(shell pkg-config --libs $(PKGS)) + +all: $(APP) + +debug: CXXFLAGS += -DDEBUG -ggdb +debug: CCFLAGS += -DDEBUG -ggdb +debug: CFLAGS += -DDEBUG -ggdb +debug: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +%.o: %.cpp $(INCS) Makefile + $(CXX) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CXX) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) + + diff --git a/deepstream-bodypose-3d/sources/deepstream-sdk/eventmsg_payload.cpp b/deepstream-bodypose-3d/sources/deepstream-sdk/eventmsg_payload.cpp new file mode 100755 index 0000000..d165bda --- /dev/null +++ b/deepstream-bodypose-3d/sources/deepstream-sdk/eventmsg_payload.cpp @@ -0,0 +1,1051 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "deepstream_schema.h" + +constexpr int NVDS_OBJECT_TYPE_PERSON_EXT_POSE = 0x103; //NVDS_OBJECT_TYPE_UNKNOWN = 102 +static std::vector> _joint_maps = { + {// Type 0 - pose2D + std::string("nose"), + std::string("neck"), + std::string("right-shoulder"), + std::string("right-elbow"), + std::string("right-hand"), + std::string("left-shoulder"), + std::string("left-elbow"), + std::string("left-hand"), + std::string("right-hip"), + std::string("right-knee"), + std::string("right-foot"), + std::string("left-hip"), + std::string("left-knee"), + std::string("left-foot"), + std::string("right-eye"), + std::string("left-eye"), + std::string("right-ear"), + std::string("left-ear"), + }, + { + std::string("pelvis"), + std::string("left-hip"), + std::string("right-hip"), + std::string("torso"), + std::string("left-knee"), + std::string("right-knee"), + std::string("neck"), + std::string("left-ankle"), + std::string("right-ankle"), + std::string("left-big-toe"), + std::string("right-big-toe"), + std::string("left-small-toe"), + std::string("right-small-toe"), + std::string("left-heel"), + std::string("right-heel"), + std::string("nose"), + std::string("left-eye"), + std::string("right-eye"), + std::string("left-ear"), + std::string("right-ear"), + std::string("left-shoulder"), + std::string("right-shoulder"), + std::string("left-elbow"), + std::string("right-elbow"), + std::string("left-wrist"), + std::string("right-wrist"), + std::string("left-pinky-knuckle"), + std::string("right-pinky-knuckle"), + std::string("left-middle-tip"), + std::string("right-middle-tip"), + std::string("left-index-knuckle"), + std::string("right-index-knuckle"), + std::string("left-thumb-tip"), + std::string("right-thumb-tip"), + }, + { + std::string("pelvis"), + std::string("left-hip"), + std::string("right-hip"), + std::string("torso"), + std::string("left-knee"), + std::string("right-knee"), + std::string("neck"), + std::string("left-ankle"), + std::string("right-ankle"), + std::string("left-big-toe"), + std::string("right-big-toe"), + std::string("left-small-toe"), + std::string("right-small-toe"), + std::string("left-heel"), + std::string("right-heel"), + std::string("nose"), + std::string("left-eye"), + std::string("right-eye"), + std::string("left-ear"), + std::string("right-ear"), + std::string("left-shoulder"), + std::string("right-shoulder"), + std::string("left-elbow"), + std::string("right-elbow"), + std::string("left-wrist"), + std::string("right-wrist"), + std::string("left-pinky-knuckle"), + std::string("right-pinky-knuckle"), + std::string("left-middle-tip"), + std::string("right-middle-tip"), + std::string("left-index-knuckle"), + std::string("right-index-knuckle"), + std::string("left-thumb-tip"), + std::string("right-thumb-tip"), + } +}; + +typedef struct NvDsJoint { + gdouble confidence; + gdouble x; + gdouble y; + gdouble z; +}NvDsJoint; + +typedef struct NvDsJoints { + + gint pose_type; + gint num_joints; + NvDsJoint *joints; + +}NvDsJoints; + +typedef struct NvDsPersonPoseExt { + gint num_poses; + NvDsJoints *poses; +}NvDsPersonPoseExt; + +static JsonObject* +generate_place_object (void *privData, NvDsEventMsgMeta *meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsPlaceObject *dsPlaceObj = NULL; + JsonObject *placeObj; + JsonObject *jobject; + JsonObject *jobject2; + + privObj = (NvDsPayloadPriv *) privData; + auto idMap = privObj->placeObj.find (meta->placeId); + + if (idMap != privObj->placeObj.end()) { + dsPlaceObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_PLACE << meta->placeId + << " in configuration file" << endl; + return NULL; + } + + /* place object + * "place": + { + "id": "string", + "name": "endeavor", + “type”: “garage”, + "location": { + "lat": 30.333, + "lon": -40.555, + "alt": 100.00 + }, + "entrance/aisle": { + "name": "walsh", + "lane": "lane1", + "level": "P2", + "coordinate": { + "x": 1.0, + "y": 2.0, + "z": 3.0 + } + } + } + */ + + placeObj = json_object_new (); + json_object_set_string_member (placeObj, "id", dsPlaceObj->id.c_str()); + json_object_set_string_member (placeObj, "name", dsPlaceObj->name.c_str()); + json_object_set_string_member (placeObj, "type", dsPlaceObj->type.c_str()); + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", dsPlaceObj->location[0]); + json_object_set_double_member (jobject, "lon", dsPlaceObj->location[1]); + json_object_set_double_member (jobject, "alt", dsPlaceObj->location[2]); + json_object_set_object_member (placeObj, "location", jobject); + + // parkingSpot / aisle /entrance sub object + jobject = json_object_new (); + + switch (meta->type) { + case NVDS_EVENT_MOVING: + case NVDS_EVENT_STOPPED: + json_object_set_string_member (jobject, "id", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "name", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "aisle", jobject); + break; + case NVDS_EVENT_EMPTY: + case NVDS_EVENT_PARKED: + json_object_set_string_member (jobject, "id", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "type", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "parkingSpot", jobject); + break; + case NVDS_EVENT_ENTRY: + case NVDS_EVENT_EXIT: + if (meta->objType == NVDS_OBJECT_TYPE_VEHICLE) { + json_object_set_string_member (jobject, "id", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "name", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "aisle", jobject); + } else { + json_object_set_string_member (jobject, "name", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "lane", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "entrance", jobject); + } + break; + default: + cout << "Event type not implemented " << endl; + break; + } + + // coordinate sub sub object + jobject2 = json_object_new (); + json_object_set_double_member (jobject2, "x", dsPlaceObj->coordinate[0]); + json_object_set_double_member (jobject2, "y", dsPlaceObj->coordinate[1]); + json_object_set_double_member (jobject2, "z", dsPlaceObj->coordinate[2]); + json_object_set_object_member (jobject, "coordinate", jobject2); + + return placeObj; +} + +static JsonObject* +generate_sensor_object (void *privData, NvDsEventMsgMeta *meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject *dsSensorObj = NULL; + JsonObject *sensorObj; + JsonObject *jobject; + + privObj = (NvDsPayloadPriv *) privData; + auto idMap = privObj->sensorObj.find (meta->sensorId); + + if (idMap != privObj->sensorObj.end()) { + dsSensorObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_SENSOR << meta->sensorId + << " in configuration file" << endl; + return NULL; + } + + /* sensor object + * "sensor": { + "id": "string", + "type": "Camera/Puck", + "location": { + "lat": 45.99, + "lon": 35.54, + "alt": 79.03 + }, + "coordinate": { + "x": 5.2, + "y": 10.1, + "z": 11.2 + }, + "description": "Entrance of Endeavor Garage Right Lane" + } + */ + + // sensor object + sensorObj = json_object_new (); + json_object_set_string_member (sensorObj, "id", dsSensorObj->id.c_str()); + json_object_set_string_member (sensorObj, "type", dsSensorObj->type.c_str()); + json_object_set_string_member (sensorObj, "description", dsSensorObj->desc.c_str()); + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", dsSensorObj->location[0]); + json_object_set_double_member (jobject, "lon", dsSensorObj->location[1]); + json_object_set_double_member (jobject, "alt", dsSensorObj->location[2]); + json_object_set_object_member (sensorObj, "location", jobject); + + // coordinate sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "x", dsSensorObj->coordinate[0]); + json_object_set_double_member (jobject, "y", dsSensorObj->coordinate[1]); + json_object_set_double_member (jobject, "z", dsSensorObj->coordinate[2]); + json_object_set_object_member (sensorObj, "coordinate", jobject); + + return sensorObj; +} + +static JsonObject* +generate_analytics_module_object (void *privData, NvDsEventMsgMeta *meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsAnalyticsObject *dsObj = NULL; + JsonObject *analyticsObj; + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->analyticsObj.find (meta->moduleId); + + if (idMap != privObj->analyticsObj.end()) { + dsObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_ANALYTICS << meta->moduleId + << " in configuration file" << endl; + return NULL; + } + + /* analytics object + * "analyticsModule": { + "id": "string", + "description": "Vehicle Detection and License Plate Recognition", + "confidence": 97.79, + "source": "OpenALR", + "version": "string" + } + */ + + // analytics object + analyticsObj = json_object_new (); + json_object_set_string_member (analyticsObj, "id", dsObj->id.c_str()); + json_object_set_string_member (analyticsObj, "description", dsObj->desc.c_str()); + json_object_set_string_member (analyticsObj, "source", dsObj->source.c_str()); + json_object_set_string_member (analyticsObj, "version", dsObj->version.c_str()); + + return analyticsObj; +} + +static JsonObject* +generate_event_object (void *privData, NvDsEventMsgMeta *meta) +{ + JsonObject *eventObj; + uuid_t uuid; + gchar uuidStr[37]; + + /* + * "event": { + "id": "event-id", + "type": "entry / exit" + } + */ + + uuid_generate_random (uuid); + uuid_unparse_lower(uuid, uuidStr); + + eventObj = json_object_new (); + json_object_set_string_member (eventObj, "id", uuidStr); + + switch (meta->type) { + case NVDS_EVENT_ENTRY: + json_object_set_string_member (eventObj, "type", "entry"); + break; + case NVDS_EVENT_EXIT: + json_object_set_string_member (eventObj, "type", "exit"); + break; + case NVDS_EVENT_MOVING: + json_object_set_string_member (eventObj, "type", "moving"); + break; + case NVDS_EVENT_STOPPED: + json_object_set_string_member (eventObj, "type", "stopped"); + break; + case NVDS_EVENT_PARKED: + json_object_set_string_member (eventObj, "type", "parked"); + break; + case NVDS_EVENT_EMPTY: + json_object_set_string_member (eventObj, "type", "empty"); + break; + case NVDS_EVENT_RESET: + json_object_set_string_member (eventObj, "type", "reset"); + break; + default: + cout << "Unknown event type " << endl; + break; + } + + return eventObj; +} + +static JsonObject* +generate_object_object (void *privData, NvDsEventMsgMeta *meta) +{ + JsonObject *objectObj; + JsonObject *jobject; + JsonObject *pobject; + guint i; + gchar tracking_id[64]; + GList *objectMask = NULL; + + // object object + objectObj = json_object_new (); + if (snprintf (tracking_id, sizeof(tracking_id), "%lu", meta->trackingId) + >= (int) sizeof(tracking_id)) + g_warning("Not enough space to copy trackingId"); + json_object_set_string_member (objectObj, "id", tracking_id); + json_object_set_double_member (objectObj, "speed", 0); + json_object_set_double_member (objectObj, "direction", 0); + json_object_set_double_member (objectObj, "orientation", 0); + + switch (meta->objType) { + case NVDS_OBJECT_TYPE_VEHICLE: + // vehicle sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsVehicleObject *dsObj = (NvDsVehicleObject *) meta->extMsg; + if (dsObj) { + json_object_set_string_member (jobject, "type", dsObj->type); + json_object_set_string_member (jobject, "make", dsObj->make); + json_object_set_string_member (jobject, "model", dsObj->model); + json_object_set_string_member (jobject, "color", dsObj->color); + json_object_set_string_member (jobject, "licenseState", dsObj->region); + json_object_set_string_member (jobject, "license", dsObj->license); + json_object_set_double_member (jobject, "confidence", meta->confidence); + } + } else { + // No vehicle object in meta data. Attach empty vehicle sub object. + json_object_set_string_member (jobject, "type", ""); + json_object_set_string_member (jobject, "make", ""); + json_object_set_string_member (jobject, "model", ""); + json_object_set_string_member (jobject, "color", ""); + json_object_set_string_member (jobject, "licenseState", ""); + json_object_set_string_member (jobject, "license", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "vehicle", jobject); + break; + case NVDS_OBJECT_TYPE_PERSON: + // person sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsPersonObject *dsObj = (NvDsPersonObject *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "apparel", dsObj->apparel); + json_object_set_double_member (jobject, "confidence", meta->confidence); + } + } else { + // No person object in meta data. Attach empty person sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "apparel", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "person", jobject); + break; + case NVDS_OBJECT_TYPE_FACE: + // face sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsFaceObject *dsObj = (NvDsFaceObject *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "glasses", dsObj->glasses); + json_object_set_string_member (jobject, "facialhair", dsObj->facialhair); + json_object_set_string_member (jobject, "name", dsObj->name); + json_object_set_string_member (jobject, "eyecolor", dsObj->eyecolor); + json_object_set_double_member (jobject, "confidence", meta->confidence); + } + } else { + // No face object in meta data. Attach empty face sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "glasses", ""); + json_object_set_string_member (jobject, "facialhair", ""); + json_object_set_string_member (jobject, "name", ""); + json_object_set_string_member (jobject, "eyecolor", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "face", jobject); + break; + case NVDS_OBJECT_TYPE_VEHICLE_EXT: + // vehicle sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsVehicleObjectExt *dsObj = (NvDsVehicleObjectExt *) meta->extMsg; + if (dsObj) { + json_object_set_string_member (jobject, "type", dsObj->type); + json_object_set_string_member (jobject, "make", dsObj->make); + json_object_set_string_member (jobject, "model", dsObj->model); + json_object_set_string_member (jobject, "color", dsObj->color); + json_object_set_string_member (jobject, "licenseState", dsObj->region); + json_object_set_string_member (jobject, "license", dsObj->license); + json_object_set_double_member (jobject, "confidence", meta->confidence); + + objectMask = dsObj->mask; + } + } else { + // No vehicle object in meta data. Attach empty vehicle sub object. + json_object_set_string_member (jobject, "type", ""); + json_object_set_string_member (jobject, "make", ""); + json_object_set_string_member (jobject, "model", ""); + json_object_set_string_member (jobject, "color", ""); + json_object_set_string_member (jobject, "licenseState", ""); + json_object_set_string_member (jobject, "license", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "vehicle", jobject); + break; + case NVDS_OBJECT_TYPE_PERSON_EXT: + // person sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsPersonObjectExt *dsObj = (NvDsPersonObjectExt *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "apparel", dsObj->apparel); + json_object_set_double_member (jobject, "confidence", meta->confidence); + + objectMask = dsObj->mask; + } + } else { + // No person object in meta data. Attach empty person sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "apparel", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "person", jobject); + break; + case NVDS_OBJECT_TYPE_FACE_EXT: + // face sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsFaceObjectExt *dsObj = (NvDsFaceObjectExt *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "glasses", dsObj->glasses); + json_object_set_string_member (jobject, "facialhair", dsObj->facialhair); + json_object_set_string_member (jobject, "name", dsObj->name); + json_object_set_string_member (jobject, "eyecolor", dsObj->eyecolor); + json_object_set_double_member (jobject, "confidence", meta->confidence); + + objectMask = dsObj->mask; + } + } else { + // No face object in meta data. Attach empty face sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "glasses", ""); + json_object_set_string_member (jobject, "facialhair", ""); + json_object_set_string_member (jobject, "name", ""); + json_object_set_string_member (jobject, "eyecolor", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "face", jobject); + break; + case NVDS_OBJECT_TYPE_UNKNOWN: + if(!meta->objectId) { + break; + } + /** No information to add; object type unknown within NvDsEventMsgMeta */ + jobject = json_object_new (); + json_object_set_object_member (objectObj, meta->objectId, jobject); + break; + case (NvDsObjectType)NVDS_OBJECT_TYPE_PERSON_EXT_POSE: + { + if (meta->extMsgSize){ + NvDsPersonPoseExt *pose_meta = (NvDsPersonPoseExt*)meta->extMsg; + + // // DEBUG + // g_message("generate_object_object(): pose_meta->num_poses = %d", pose_meta->num_poses); + + for (int i=0; inum_poses; i++) { + // bbox sub object + jobject = json_object_new (); + json_object_set_int_member (jobject, "topleftx", meta->bbox.left); + json_object_set_int_member (jobject, "toplefty", meta->bbox.top); + json_object_set_int_member (jobject, "bottomrightx", meta->bbox.left + meta->bbox.width); + json_object_set_int_member (jobject, "bottomrighty", meta->bbox.top + meta->bbox.height); + json_object_set_object_member (objectObj, "bbox", jobject); + + pobject = json_object_new (); + int joint_index = 0; + + for (joint_index = 0;joint_index < pose_meta->poses[i].num_joints; joint_index++) + { + if (pose_meta->poses[i].joints[joint_index].confidence > 0.0) + { + jobject = json_object_new (); + std::string s = _joint_maps[pose_meta->poses[i].pose_type][joint_index]; + char *json_name = const_cast(s.c_str()); + + if (pose_meta->poses[i].pose_type==0) { + json_object_set_int_member (jobject, "x", pose_meta->poses[i].joints[joint_index].x); + json_object_set_int_member (jobject, "y", pose_meta->poses[i].joints[joint_index].y); + json_object_set_int_member (jobject, "confidence", pose_meta->poses[i].joints[joint_index].confidence); + json_object_set_object_member (pobject, reinterpret_cast(json_name), jobject); + } + else if ((pose_meta->poses[i].pose_type==1)||(pose_meta->poses[i].pose_type==2)) { + // pose3d or pose25d from BodyPose3DNet + json_object_set_int_member (jobject, "x", pose_meta->poses[i].joints[joint_index].x); + json_object_set_int_member (jobject, "y", pose_meta->poses[i].joints[joint_index].y); + json_object_set_int_member (jobject, "z", pose_meta->poses[i].joints[joint_index].z); + json_object_set_int_member (jobject, "confidence", pose_meta->poses[i].joints[joint_index].confidence); + json_object_set_object_member (pobject, reinterpret_cast(json_name), jobject); + } + + } + } + + if (pose_meta->poses[i].pose_type==0) + json_object_set_object_member (objectObj, "pose2D", pobject); + else if (pose_meta->poses[i].pose_type==1) + json_object_set_object_member (objectObj, "pose3D", pobject); + else if (pose_meta->poses[i].pose_type==2) + json_object_set_object_member (objectObj, "pose25D", pobject); + } + } + } + break; + default: + cout << "Object type not implemented" << endl; + break; + } + + + if (objectMask) { + GList *l; + JsonArray *maskArray = json_array_sized_new (g_list_length(objectMask)); + + for (l = objectMask; l != NULL; l = l->next) { + GArray *polygon = (GArray *) l->data; + JsonArray *polygonArray = json_array_sized_new (polygon->len); + + for (i = 0; i < polygon->len; i++) { + gdouble value = g_array_index (polygon, gdouble, i); + + json_array_add_double_element (polygonArray, value); + } + + json_array_add_array_element (maskArray, polygonArray); + } + + json_object_set_array_member (objectObj, "maskoutline", maskArray); + } + + // signature sub array + if (meta->objSignature.size) { + JsonArray *jArray = json_array_sized_new (meta->objSignature.size); + + for (i = 0; i < meta->objSignature.size; i++) { + json_array_add_double_element (jArray, meta->objSignature.signature[i]); + } + json_object_set_array_member (objectObj, "signature", jArray); + } + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", meta->location.lat); + json_object_set_double_member (jobject, "lon", meta->location.lon); + json_object_set_double_member (jobject, "alt", meta->location.alt); + json_object_set_object_member (objectObj, "location", jobject); + + // coordinate sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "x", meta->coordinate.x); + json_object_set_double_member (jobject, "y", meta->coordinate.y); + json_object_set_double_member (jobject, "z", meta->coordinate.z); + json_object_set_object_member (objectObj, "coordinate", jobject); + + return objectObj; +} + +gchar* generate_event_message (void *privData, NvDsEventMsgMeta *meta) +{ + JsonNode *rootNode; + JsonObject *rootObj; + JsonObject *placeObj; + JsonObject *sensorObj; + JsonObject *analyticsObj; + JsonObject *eventObj; + JsonObject *objectObj; + gchar *message; + + uuid_t msgId; + gchar msgIdStr[37]; + + uuid_generate_random (msgId); + uuid_unparse_lower(msgId, msgIdStr); + + // place object + placeObj = generate_place_object (privData, meta); + + // sensor object + sensorObj = generate_sensor_object (privData, meta); + + // analytics object + analyticsObj = generate_analytics_module_object (privData, meta); + + // object object + objectObj = generate_object_object (privData, meta); + + // event object + eventObj = generate_event_object (privData, meta); + + // root object + rootObj = json_object_new (); + json_object_set_string_member (rootObj, "messageid", msgIdStr); + json_object_set_string_member (rootObj, "mdsversion", "1.0"); + json_object_set_string_member (rootObj, "@timestamp", meta->ts); + json_object_set_object_member (rootObj, "place", placeObj); + json_object_set_object_member (rootObj, "sensor", sensorObj); + json_object_set_object_member (rootObj, "analyticsModule", analyticsObj); + json_object_set_object_member (rootObj, "object", objectObj); + json_object_set_object_member (rootObj, "event", eventObj); + + if (meta->videoPath) + json_object_set_string_member (rootObj, "videoPath", meta->videoPath); + else + json_object_set_string_member (rootObj, "videoPath", ""); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, rootObj); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (rootObj); + + return message; +} + +static const gchar* +object_enum_to_str (NvDsObjectType type, gchar* objectId) +{ + switch (type) { + case NVDS_OBJECT_TYPE_VEHICLE: + return "Vehicle"; + case NVDS_OBJECT_TYPE_FACE: + return "Face"; + case NVDS_OBJECT_TYPE_PERSON: + case (NvDsObjectType)NVDS_OBJECT_TYPE_PERSON_EXT_POSE: + return "Person"; + case NVDS_OBJECT_TYPE_BAG: + return "Bag"; + case NVDS_OBJECT_TYPE_BICYCLE: + return "Bicycle"; + case NVDS_OBJECT_TYPE_ROADSIGN: + return "RoadSign"; + case NVDS_OBJECT_TYPE_CUSTOM: + return "Custom"; + case NVDS_OBJECT_TYPE_UNKNOWN: + return objectId ? objectId : "Unknown"; + default: + return "Unknown"; + } +} + +static const gchar* +to_str (gchar* cstr) +{ + return reinterpret_cast(cstr) ? cstr : ""; +} + +static const gchar * +sensor_id_to_str (void *privData, gint sensorId) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject *dsObj = NULL; + + g_return_val_if_fail (privData, NULL); + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->sensorObj.find (sensorId); + if (idMap != privObj->sensorObj.end()) { + dsObj = &idMap->second; + return dsObj->id.c_str(); + } else { + cout << "No entry for " CONFIG_GROUP_SENSOR << sensorId + << " in configuration file" << endl; + return NULL; + } +} + +static void +generate_mask_array (NvDsEventMsgMeta *meta, JsonArray *jArray, GList *mask) +{ + unsigned int i; + GList *l; + stringstream ss; + bool started = false; + + ss << meta->trackingId << "|" << g_list_length(mask); + + for (l = mask; l != NULL; l = l->next) { + GArray *polygon = (GArray *) l->data; + + if (started) + ss << "|#"; + + started = true; + + for (i = 0; i < polygon->len; i++) { + gdouble value = g_array_index (polygon, gdouble, i); + ss << "|" << value; + } + } + json_array_add_string_element (jArray, ss.str().c_str()); +} + +gchar* generate_event_message_minimal (void *privData, NvDsEvent *events, guint size) +{ + /* + The JSON structure of the frame + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + ".......object-1 attributes...........", + ".......object-2 attributes...........", + ".......object-3 attributes..........." + ] + } + */ + + /* + An example object with Vehicle object-type + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + "957|1834|150|1918|215|Vehicle|#|sedan|Bugatti|M|blue|CA 444|California|0.8", + "..........." + ] + } + */ + + JsonNode *rootNode; + JsonObject *jobject; + JsonArray *jArray; + JsonArray *maskArray = NULL; + guint i; + stringstream ss; + gchar *message = NULL; + + jArray = json_array_new (); + + for (i = 0; i < size; i++) { + GList *objectMask = NULL; + + ss.str(""); + ss.clear(); + + NvDsEventMsgMeta *meta = events[i].metadata; + ss << meta->trackingId << "|" << meta->bbox.left << "|" << meta->bbox.top + << "|" << meta->bbox.left + meta->bbox.width << "|" << meta->bbox.top + meta->bbox.height + << "|" << object_enum_to_str (meta->objType, meta->objectId); + + if (meta->extMsg && meta->extMsgSize) { + // Attach secondary inference attributes. + switch (meta->objType) { + case NVDS_OBJECT_TYPE_VEHICLE: { + NvDsVehicleObject *dsObj = (NvDsVehicleObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->type) << "|" << to_str(dsObj->make) << "|" + << to_str(dsObj->model) << "|" << to_str(dsObj->color) << "|" << to_str(dsObj->license) + << "|" << to_str(dsObj->region) << "|" << meta->confidence; + } + } + break; + case (NvDsObjectType)NVDS_OBJECT_TYPE_PERSON_EXT_POSE: { + NvDsPersonPoseExt *pose_meta = (NvDsPersonPoseExt *) meta->extMsg; + if (pose_meta) { + // STUB person + ss << "|#||||||" << meta->confidence; + + for (int i = 0; i < pose_meta->num_poses; i++) { + int type = pose_meta->poses[i].pose_type; + + if (type == 0) { + ss << "|#|pose2D|"; + for (int joint_index = 0; joint_index < pose_meta->poses[i].num_joints; joint_index++) { + std::string s = _joint_maps[type][joint_index] + + "," + std::to_string(pose_meta->poses[i].joints[joint_index].x) + + "," + to_string(pose_meta->poses[i].joints[joint_index].y) + + "," + to_string(pose_meta->poses[i].joints[joint_index].confidence); + ss << s << "|"; + } + } + else if (type == 1) {// BodyPose3DNet + ss << "|#|pose3D|"; + for (int joint_index = 0; joint_index < pose_meta->poses[i].num_joints; joint_index++) { + std::string s = _joint_maps[type][joint_index] + + "," + std::to_string(pose_meta->poses[i].joints[joint_index].x) + + "," + to_string(pose_meta->poses[i].joints[joint_index].y) + + "," + to_string(pose_meta->poses[i].joints[joint_index].z) + + "," + to_string(pose_meta->poses[i].joints[joint_index].confidence); + ss << s << "|"; + } + } + else if (type == 2) {// BodyPose3DNet + ss << "|#|pose25D|"; + for (int joint_index = 0; joint_index < pose_meta->poses[i].num_joints; joint_index++) { + std::string s = _joint_maps[type][joint_index] + + "," + std::to_string(pose_meta->poses[i].joints[joint_index].x) + + "," + to_string(pose_meta->poses[i].joints[joint_index].y) + + "," + to_string(pose_meta->poses[i].joints[joint_index].z) + + "," + to_string(pose_meta->poses[i].joints[joint_index].confidence); + ss << s << "|"; + } + } + } + } + } + break; + case NVDS_OBJECT_TYPE_PERSON: { + NvDsPersonObject *dsObj = (NvDsPersonObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->apparel) + << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_FACE: { + NvDsFaceObject *dsObj = (NvDsFaceObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->glasses) + << "|" << to_str(dsObj->facialhair) << "|" << to_str(dsObj->name) << "|" + << "|" << to_str(dsObj->eyecolor) << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_VEHICLE_EXT: { + NvDsVehicleObjectExt *dsObj = (NvDsVehicleObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->type) << "|" << to_str(dsObj->make) << "|" + << to_str(dsObj->model) << "|" << to_str(dsObj->color) << "|" << to_str(dsObj->license) + << "|" << to_str(dsObj->region) << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + case NVDS_OBJECT_TYPE_PERSON_EXT: { + NvDsPersonObjectExt *dsObj = (NvDsPersonObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->apparel) + << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + case NVDS_OBJECT_TYPE_FACE_EXT: { + NvDsFaceObjectExt *dsObj = (NvDsFaceObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->glasses) + << "|" << to_str(dsObj->facialhair) << "|" << to_str(dsObj->name) << "|" + << "|" << to_str(dsObj->eyecolor) << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + default: + cout << "Object type (" << meta->objType << ") not implemented" << endl; + break; + } + } + + if (objectMask) { + if (maskArray == NULL) + maskArray = json_array_new (); + generate_mask_array (meta, maskArray, objectMask); + } + + json_array_add_string_element (jArray, ss.str().c_str()); + } + + // It is assumed that all events / objects are associated with same frame. + // Therefore ts / sensorId / frameId of first object can be used. + + jobject = json_object_new (); + json_object_set_string_member (jobject, "version", "4.0"); + json_object_set_int_member (jobject, "id", events[0].metadata->frameId); + json_object_set_string_member (jobject, "@timestamp", events[0].metadata->ts); + if (events[0].metadata->sensorStr) { + json_object_set_string_member (jobject, "sensorId", events[0].metadata->sensorStr); + } else if ((NvDsPayloadPriv *) privData) { + json_object_set_string_member (jobject, "sensorId", + to_str((gchar *) sensor_id_to_str (privData, events[0].metadata->sensorId))); + } else { + json_object_set_string_member (jobject, "sensorId", "0"); + } + + json_object_set_array_member (jobject, "objects", jArray); + if (maskArray && json_array_get_length (maskArray) > 0) + json_object_set_array_member (jobject, "masks", maskArray); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, jobject); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (jobject); + + return message; +} + diff --git a/deepstream-bodypose-3d/sources/deepstream_pose_estimation_app.cpp b/deepstream-bodypose-3d/sources/deepstream_pose_estimation_app.cpp new file mode 100755 index 0000000..a3db65c --- /dev/null +++ b/deepstream-bodypose-3d/sources/deepstream_pose_estimation_app.cpp @@ -0,0 +1,2062 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "cuda_runtime_api.h" +#include "gstnvdsinfer.h" +#include "gstnvdsmeta.h" +#include "nvdsgstutils.h" +#include "nvbufsurface.h" +#include "nvdsmeta_schema.h" +#include "deepstream_common.h" +#include "deepstream_perf.h" +#include "deepstream_app_version.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +GST_DEBUG_CATEGORY_STATIC (NVDS_APP); // define category (statically) +#define GST_CAT_DEFAULT NVDS_APP // set as default + +#define EPS 1e-6 +#define MAX_TIME_STAMP_LEN 32 +#define MAX_DISPLAY_LEN 64 + +#define PGIE_CLASS_ID_VEHICLE 0 +#define PGIE_CLASS_ID_PERSON 2 +#define PGIE_CLASS_ID_PRODUCT 4 + +// Default camera attributes +#define MUXER_OUTPUT_WIDTH 1280 +#define MUXER_OUTPUT_HEIGHT 720 +#define FOCAL_LENGTH 800.79041f + +/* Padding due to AR SDK model requires bigger bboxes*/ +#define PAD_DIM 128 + +/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set + * based on the fastest source's framerate. */ +#define MUXER_BATCH_TIMEOUT_USEC 40000 + +/* NVIDIA Decoder source pad memory feature. This feature signifies that source + * pads having this capability will push GstBuffers containing cuda buffers. */ +#define GST_CAPS_FEATURES_NVMM "memory:NVMM" +#define CONFIG_GPU_ID "gpu-id" + +#define PGIE_CONFIG_FILE "../configs/config_infer_primary_peoplenet.txt" +#define SGIE_CONFIG_FILE "../configs/config_infer_secondary_bodypose3dnet.txt" +#define TRACKER_CONFIG_FILE "../configs/config_tracker.txt" +#define CONFIG_GROUP_TRACKER "tracker" +#define CONFIG_GROUP_TRACKER_WIDTH "tracker-width" +#define CONFIG_GROUP_TRACKER_HEIGHT "tracker-height" +#define CONFIG_GROUP_TRACKER_LL_CONFIG_FILE "ll-config-file" +#define CONFIG_GROUP_TRACKER_LL_LIB_FILE "ll-lib-file" +#define CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS "enable-batch-process" +#define MAX_TRACKING_ID_LEN 16 + +#define CHECK_ERROR(error) \ + if (error) { \ + g_printerr ("Error while parsing config file: %s\n", error->message); \ + goto done; \ + } + +//---Global variables derived from program arguments--- +static gboolean _print_version = FALSE; +static gboolean _print_dependencies_version = FALSE; +static gboolean _print_fps = FALSE; +static gchar *_input = NULL; +static gchar *_output = NULL; +static gchar *_nvmsgbroker_conn_str = NULL; +static gchar *_pose_filename = NULL; +static gchar *_tracker = NULL; +static gchar *_publish_pose = NULL; +static guint _cintr = FALSE; +static gboolean _quit = FALSE; +FILE *_pose_file = NULL; +double _focal_length_dbl = FOCAL_LENGTH; +float _focal_length = (float)_focal_length_dbl; +int _image_width = MUXER_OUTPUT_WIDTH; +int _image_height = MUXER_OUTPUT_HEIGHT; +int _pad_dim = PAD_DIM;// A scaled version of PAD_DIM +Eigen::Matrix3f _K;// Camera intrinsic matrix +//---Global variables derived from program arguments--- + +static GstElement *pipeline = NULL; +static gint _fps_interval=1; +static gint _osd_process_mode = 0; + +gint frame_number = 0; + +#if (DS_VERSION_MAJOR < 6) || ((DS_VERSION_MAJOR == 6) && (DS_VERSION_MINOR < 2)) +typedef struct NvDsJoint { + gdouble confidence; + gdouble x; + gdouble y; + gdouble z; +}NvDsJoint; + +typedef struct NvDsJoints { + + gint pose_type; + gint num_joints; + NvDsJoint *joints; + +}NvDsJoints; +#endif + +typedef struct NvDsPersonPoseExt { + gint num_poses; + NvDsJoints *poses; +}NvDsPersonPoseExt; + + +class OneEuroFilter { +public: + /// Default constructor + OneEuroFilter() { + reset(30.0f /* Hz */, 0.1f /* Hz */, 0.09f /* ??? */, 0.5f /* Hz */); + } + /// Constructor + /// @param dataUpdateRate the sampling rate, i.e. the number of samples per unit of time. + /// @param minCutoffFreq the lowest bandwidth filter applied. + /// @param cutoffSlope the rate at which the filter adapts: higher levels reduce lag. + /// @param derivCutoffFreq the bandwidth of the filter applied to smooth the derivative, default 1 Hz. + OneEuroFilter(float dataUpdateRate, float minCutoffFreq, float cutoffSlope, float derivCutoffFreq) { + reset(dataUpdateRate, minCutoffFreq, cutoffSlope, derivCutoffFreq); + } + /// Reset all parameters of the filter. + /// @param dataUpdateRate the sampling rate, i.e. the number of samples per unit of time. + /// @param minCutoffFreq the lowest bandwidth filter applied. + /// @param cutoffSlope the rate at which the filter adapts: higher levels reduce lag. + /// @param derivCutoffFreq the bandwidth of the filter applied to smooth the derivative, default 1 Hz. + void reset(float dataUpdateRate, float minCutoffFreq, float cutoffSlope, float derivCutoffFreq) { + reset(); _rate = dataUpdateRate; _minCutoff = minCutoffFreq; _beta = cutoffSlope; _dCutoff = derivCutoffFreq; + } + /// Reset only the initial condition of the filter, leaving parameters the same. + void reset() { _firstTime = true; _xFilt.reset(); _dxFilt.reset(); } + /// Apply the one euro filter to the given input. + /// @param x the unfiltered input value. + /// @return the filtered output value. + float filter(float x) + { + float dx, edx, cutoff; + if (_firstTime) { + _firstTime = false; + dx = 0; + } else { + dx = (x - _xFilt.hatXPrev()) * _rate; + } + edx = _dxFilt.filter(dx, alpha(_rate, _dCutoff)); + cutoff = _minCutoff + _beta * fabsf(edx); + return _xFilt.filter(x, alpha(_rate, cutoff)); + } + + +private: + class LowPassFilter { + public: + LowPassFilter() { reset(); } + void reset() { _firstTime = true; } + float hatXPrev() const { return _hatXPrev; } + float filter(float x, float alpha){ + if (_firstTime) { + _firstTime = false; + _hatXPrev = x; + } + float hatX = alpha * x + (1.f - alpha) * _hatXPrev; + _hatXPrev = hatX; + return hatX; + + } + private: + float _hatXPrev; + bool _firstTime; + }; + inline float alpha(float rate, float cutoff) { + const float kOneOverTwoPi = 0.15915494309189533577f; // 1 / (2 * pi) + // The paper has 4 divisions, but we only use one + // float tau = kOneOverTwoPi / cutoff, te = 1.f / rate; + // return 1.f / (1.f + tau / te); + return cutoff / (rate * kOneOverTwoPi + cutoff); +} + bool _firstTime; + float _rate, _minCutoff, _dCutoff, _beta; + LowPassFilter _xFilt, _dxFilt; +}; + +//===Global variables=== +std::unordered_map> g_filter_pose25d; +OneEuroFilter m_filterRootDepth; // Root node in pose25d. + +fpos_t g_fp_25_pos; + +//===Global variables=== + +#define ACQUIRE_DISP_META(dmeta) \ + if (dmeta->num_circles == MAX_ELEMENTS_IN_DISPLAY_META || \ + dmeta->num_labels == MAX_ELEMENTS_IN_DISPLAY_META || \ + dmeta->num_lines == MAX_ELEMENTS_IN_DISPLAY_META) \ + { \ + dmeta = nvds_acquire_display_meta_from_pool(bmeta);\ + nvds_add_display_meta_to_frame(frame_meta, dmeta);\ + }\ + +#define GET_LINE(lparams) \ + ACQUIRE_DISP_META(dmeta)\ + lparams = &dmeta->line_params[dmeta->num_lines];\ + dmeta->num_lines++;\ + +static float _sgie_classifier_threshold = FLT_MIN; + +typedef struct NvAR_Point3f { + float x, y, z; +} NvAR_Point3f; + +static Eigen::Matrix3f m_K_inv_transpose; +const float m_scale_ll[] = { + 0.5000, 0.5000, 1.0000, 0.8175, 0.9889, 0.2610, 0.7942, 0.5724, 0.5078, + 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3433, 0.8171, + 0.9912, 0.2610, 0.8259, 0.5724, 0.5078, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0000, 0.0000, 0.3422, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000}; +const float m_mean_ll[] = { + 246.3427f, 246.3427f, 492.6854f, 402.4380f, 487.0321f, 128.6856f, 391.6295f, + 281.9928f, 249.9478f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, + 0.0000f, 0.0000f, 169.1832f, 402.2611f, 488.1824f, 128.6848f, 407.5836f, + 281.9897f, 249.9489f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, + 0.0000f, 0.0000f, 168.6137f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, + 0.0000f}; + +/* Given 2D and ZRel, we need to find the depth of the root to reconstruct the scale normalized 3D Pose. + While there exists many 3D poses that can have the same 2D projection, given the 2.5D pose and intrinsic camera parameters, + there exists a unique 3D pose that satisfies (Xˆn − Xˆm)**2 + (Yˆn − Yˆm)**2 + (Zˆn − Zˆm)**2 = C**2. + Refer Section 3.3 of https://arxiv.org/pdf/1804.09534.pdf for more details. +*/ +std::vector calculateZRoots(const std::vector& X0, const std::vector& X1, + const std::vector& Y0, const std::vector& Y1, + const std::vector& Zrel0, + const std::vector& Zrel1, const std::vector& C) { + std::vector zRoots(X0.size()); + for (int i = 0; i < X0.size(); i++) { + double x0 = (double)X0[i], x1 = (double)X1[i], y0 = (double)Y0[i], y1 = (double)Y1[i], + z0 = (double)Zrel0[i], z1 = (double)Zrel1[i]; + double a = ((x1 - x0) * (x1 - x0)) + ((y1 - y0) * (y1 - y0)); + double b = 2 * (z1 * ((x1 * x1) + (y1 * y1) - x1 * x0 - y1 * y0) + + z0 * ((x0 * x0) + (y0 * y0) - x1 * x0 - y1 * y0)); + double c = ((x1 * z1 - x0 * z0) * (x1 * z1 - x0 * z0)) + + ((y1 * z1 - y0 * z0) * (y1 * z1 - y0 * z0)) + + ((z1 - z0) * (z1 - z0)) - (C[i] * C[i]); + double d = (b * b) - (4 * a * c); + + // make sure the solutions are valid + a = fmax(DBL_EPSILON, a); + d = fmax(DBL_EPSILON, d); + zRoots[i] = (float) ((-b + sqrt(d)) / (2 * a + 1e-8)); + } + return zRoots; +} + +float median(std::vector& v) { + size_t n = v.size() / 2; + nth_element(v.begin(), v.begin() + n, v.end()); + return v[n]; +} + +/* Given 2D keypoints and the relative depth of each keypoint w.r.t the root, we find the depth of the root + to reconstruct the scale normalized 3D pose. +*/ +std::vector liftKeypoints25DTo3D(const float* p2d, + const float* pZRel, + const int numKeypoints, + const Eigen::Matrix3f& KInv, + const float limbLengths[]) { + + const int ROOT = 0; + + // Contains the relative depth values of each keypoints + std::vector zRel(numKeypoints, 0.f); + + // Matrix containing the 2D keypoints. + Eigen::MatrixXf XY1 = Eigen::MatrixXf(numKeypoints, 3); + + // Mean distance between a specific pair and its parent. + std::vector C; + + // Indices representing keypoints and its parents for limb lengths > 0. + // In our dataset, we only have limb length information for few keypoints. + std::vector idx0 = { 0, 3, 6, 8, 5, 2, 2, 21, 23, 21, 7, 4, 1, 1, 20, 22, 20 }; + std::vector idx1 = { 3, 6, 0, 5, 2, 0, 21, 23, 25, 6, 4, 1, 0, 20, 22, 24, 6 }; + + std::vector X0(idx0.size(), 0.f), Y0(idx0.size(), 0.f), X1(idx0.size(), 0.f), Y1(idx0.size(), 0.f), + zRel0(idx0.size(), 0.f), zRel1(idx0.size(), 0.f); + + for (int i = 0; i < numKeypoints; i++) { + zRel[i] = pZRel[i]; + + XY1.row(i) << p2d[i * 2], p2d[(i * 2) + 1], 1.f; + + if (limbLengths[i] > 0.f) C.push_back(limbLengths[i]); + } + + // Set relative depth of root to be 0 as the relative depth is measure w.r.t the root. + zRel[ROOT] = 0.f; + + for (int i = 0; i < XY1.rows(); i++) { + float x = XY1(i, 0); + float y = XY1(i, 1); + float z = XY1(i, 2); + XY1.row(i) << x, y, z; + } + + XY1 = XY1 * KInv; + + for (int i = 0; i < idx0.size(); i++) { + X0[i] = XY1(idx0[i], 0); + Y0[i] = XY1(idx0[i], 1); + X1[i] = XY1(idx1[i], 0); + Y1[i] = XY1(idx1[i], 1); + zRel0[i] = zRel[idx0[i]]; + zRel1[i] = zRel[idx1[i]]; + } + + std::vector zRoots = calculateZRoots(X0, X1, Y0, Y1, zRel0, zRel1, C); + + float zRootsMedian = median(zRoots); + + zRootsMedian = m_filterRootDepth.filter(zRootsMedian); + + std::vector p3d(numKeypoints, { 0.f, 0.f, 0.f }); + + for (int i = 0; i < numKeypoints; i++) { + p3d[i].x = XY1(i, 0) * (zRel[i] + zRootsMedian); + p3d[i].y = XY1(i, 1) * (zRel[i] + zRootsMedian); + p3d[i].z = XY1(i, 2) * (zRel[i] + zRootsMedian); + } + + return p3d; +} + +/* Once we have obtained the scale normalized 3D pose, we use the mean limb lengths of keypoint-keypointParent pairs +* to find the scale of the whole body. We solve for +* s^ = argmin sum((s * L2_norm(P_k - P_l) - meanLimbLength_k_l)**2), solve for s. +* meanLimbLength_k_l = mean length of the bone between keypoints k and l in the training data +* P_k and P_l are the keypoint location of k and l. +* +* We have a least squares minimization, where we are trying to minimize the magnitude of the error: + (target - scale * unit_length). Thus, we're minimizing T - sL. By the normal equations, the optimal solution is: + s = inv([L'L]) * L'T + +*/ +float recoverScale(const std::vector& p3d, const float* scores, + const float targetLengths[]) { + std::vector validIdx; + + // Indices of keypoints for which we have the length information. + for (int i = 0; i < p3d.size(); i++) { + if (targetLengths[i] > 0.f) validIdx.push_back(i); + } + + Eigen::MatrixXf targetLenMatrix = Eigen::MatrixXf(validIdx.size(), 1); + + for (int i = 0; i < validIdx.size(); i++) { + targetLenMatrix(i, 0) = targetLengths[validIdx[i]]; + } + + // Indices representing keypoints and its parents for limb lengths > 0. + // In our dataset, we have only have limb length information for few keypoints. + std::vector idx0 = { 0, 3, 6, 8, 5, 2, 2, 21, 23, 21, 7, 4, 1, 1, 20, 22, 20 }; + std::vector idx1 = { 3, 6, 0, 5, 2, 0, 21, 23, 25, 6, 4, 1, 0, 20, 22, 24, 6 }; + + Eigen::MatrixXf unitLength = Eigen::MatrixXf(idx0.size(), 1); + Eigen::VectorXf limbScores(unitLength.size()); + float squareNorms = 0.f; + float limbScoresSum = 0.f; + for (int i = 0; i < idx0.size(); i++) { + unitLength(i, 0) = sqrtf((p3d[idx0[i]].x - p3d[idx1[i]].x) * (p3d[idx0[i]].x - p3d[idx1[i]].x) + + (p3d[idx0[i]].y - p3d[idx1[i]].y) * (p3d[idx0[i]].y - p3d[idx1[i]].y) + + (p3d[idx0[i]].z - p3d[idx1[i]].z) * (p3d[idx0[i]].z - p3d[idx1[i]].z)); + + limbScores[i] = scores[idx0[i]] * scores[idx1[i]]; + limbScoresSum += limbScores[i]; + } + + for (int i = 0; i < limbScores.size(); i++) { + limbScores[i] /= limbScoresSum; + squareNorms += ((unitLength(i, 0) * unitLength(i, 0)) * limbScores[i]); + } + + auto limbScoreDiag = limbScores.asDiagonal(); + + //Eigen::MatrixXf numerator1 = ; + Eigen::MatrixXf numerator = (unitLength.transpose() * limbScoreDiag) * targetLenMatrix; + + return numerator(0, 0) / squareNorms; +} + +void generate_ts_rfc3339 (char *buf, int buf_size) +{ + time_t tloc; + struct tm tm_log; + struct timespec ts; + char strmsec[6]; //.nnnZ\0 + + clock_gettime(CLOCK_REALTIME, &ts); + memcpy(&tloc, (void *)(&ts.tv_sec), sizeof(time_t)); + gmtime_r(&tloc, &tm_log); + strftime(buf, buf_size,"%Y-%m-%dT%H:%M:%S", &tm_log); + int ms = ts.tv_nsec/1000000; + g_snprintf(strmsec, sizeof(strmsec),".%.3dZ", ms); + strncat(buf, strmsec, buf_size); +} + +static +gpointer copy_bodypose_meta (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + NvDsEventMsgMeta *dstMeta = NULL; + NvDsPersonObject *srcExt = NULL; + NvDsPersonObject *dstExt = NULL; + + dstMeta = (NvDsEventMsgMeta *)g_memdup ((gpointer)srcMeta, sizeof(NvDsEventMsgMeta)); + + // pose + dstMeta->pose.num_joints = srcMeta->pose.num_joints; + dstMeta->pose.pose_type = srcMeta->pose.pose_type; + dstMeta->pose.joints = (NvDsJoint *)g_memdup ((gpointer)srcMeta->pose.joints, + sizeof(NvDsJoint)*srcMeta->pose.num_joints); + + if (srcMeta->ts) + dstMeta->ts = g_strdup (srcMeta->ts); + + if (srcMeta->sensorStr) + dstMeta->sensorStr = g_strdup (srcMeta->sensorStr); + + if (srcMeta->objSignature.size > 0) { + dstMeta->objSignature.signature = (gdouble *)g_memdup ((gpointer)srcMeta->objSignature.signature, + sizeof(gdouble)*srcMeta->objSignature.size); + dstMeta->objSignature.size = srcMeta->objSignature.size; + } + + if(srcMeta->objectId) { + dstMeta->objectId = g_strdup (srcMeta->objectId); + } + + if (srcMeta->extMsg){ + dstMeta->extMsg = g_memdup(srcMeta->extMsg, srcMeta->extMsgSize); + dstMeta->extMsgSize = srcMeta->extMsgSize; + srcExt = (NvDsPersonObject *)srcMeta->extMsg; + dstExt = (NvDsPersonObject *)dstMeta->extMsg; + dstExt->gender = g_strdup(srcExt->gender); + dstExt->hair = g_strdup(srcExt->hair); + dstExt->cap = g_strdup(srcExt->cap); + dstExt->apparel = g_strdup(srcExt->apparel); + dstExt->age = srcExt->age; + } + return dstMeta; +} + +static void +release_bodypose_meta (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + + // pose + g_free (srcMeta->pose.joints); + g_free (srcMeta->ts); + g_free (srcMeta->sensorStr); + + if (srcMeta->objSignature.size > 0) { + g_free (srcMeta->objSignature.signature); + srcMeta->objSignature.size = 0; + } + + if(srcMeta->objectId) { + g_free (srcMeta->objectId); + } + + g_free (srcMeta->extMsg); + srcMeta->extMsgSize = 0; + srcMeta->extMsg = NULL; + + g_free (user_meta->user_meta_data); + user_meta->user_meta_data = NULL; +} + +void build_msg_meta(NvDsFrameMeta *frame_meta, + NvDsObjectMeta *obj_meta, + const int numKeyPoints, + const float *keypoints, + const float *keypointsZRel, + const float *keypoints_confidence, + const std::vector &p3dLifted) +{ + NvDsEventMsgMeta *msg_meta = (NvDsEventMsgMeta *) g_malloc0 (sizeof (NvDsEventMsgMeta)); + NvDsPersonObject *msg_meta_ext = (NvDsPersonObject *) g_malloc0 (sizeof (NvDsPersonObject)); + + msg_meta->type = NVDS_EVENT_ENTRY; //Should this be ENTRY + msg_meta->objType = (NvDsObjectType) NVDS_OBJECT_TYPE_PERSON; + msg_meta->bbox.top = obj_meta->rect_params.top; + msg_meta->bbox.left = obj_meta->rect_params.left; + msg_meta->bbox.width = obj_meta->rect_params.width; + msg_meta->bbox.height = obj_meta->rect_params.height; + msg_meta->extMsg = msg_meta_ext; + msg_meta->extMsgSize = sizeof(NvDsPersonObject); + msg_meta_ext->gender = g_strdup(""); + msg_meta_ext->hair = g_strdup(""); + msg_meta_ext->cap = g_strdup(""); + msg_meta_ext->apparel = g_strdup(""); + msg_meta_ext->age = 0; + + //---msg_meta->poses--- + if (1) { + int pose_types[8]; + + if (_publish_pose) { + if (!strcmp(_publish_pose, "pose3d")) + msg_meta->pose.pose_type = 2;// pdatapose3D + else if (!strcmp(_publish_pose, "pose25d")) + msg_meta->pose.pose_type = 1;// pdatapose3D + } + else { + msg_meta->pose.pose_type = 2;// pdatapose3D + } + + msg_meta->pose.num_joints = numKeyPoints; + msg_meta->pose.joints = (NvDsJoint *)g_malloc0(sizeof(NvDsJoint) * numKeyPoints); + + if (msg_meta->pose.pose_type == 0) {// pose25d without zRel + for(int i = 0; i < msg_meta->pose.num_joints; i++){ + msg_meta->pose.joints[i].x = keypoints[2*i ]; + msg_meta->pose.joints[i].y = keypoints[2*i+1]; + msg_meta->pose.joints[i].confidence = keypoints_confidence[i]; + } + } + else if (msg_meta->pose.pose_type == 2) {// pose3d + for(int i = 0; i < msg_meta->pose.num_joints; i++){ + msg_meta->pose.joints[i].x = p3dLifted[i].x; + msg_meta->pose.joints[i].y = p3dLifted[i].y; + msg_meta->pose.joints[i].z = p3dLifted[i].z; + msg_meta->pose.joints[i].confidence = keypoints_confidence[i]; + } + } + else if (msg_meta->pose.pose_type == 1) {// pose25d + for(int i = 0; i < msg_meta->pose.num_joints; i++){ + msg_meta->pose.joints[i].x = keypoints[2*i ]; + msg_meta->pose.joints[i].y = keypoints[2*i+1]; + msg_meta->pose.joints[i].z = keypointsZRel[i]; + msg_meta->pose.joints[i].confidence = keypoints_confidence[i]; + } + } + } + // // DEBUG + // g_message("Metadata poses are built.\n"); + //---msg_meta->poses--- + // msg_meta->embedding = + // msg_meta->location = + // msg_meta->coordinate = + // msg_meta->objSignature = + //msg_meta->objClassId = PGIE_CLASS_ID_PERSON; + msg_meta->objClassId = obj_meta->class_id; + // msg_meta->sensorId = + // msg_meta->moduleId = + // msg_meta->placeId = + // msg_meta->componentId = + msg_meta->frameId = frame_meta->frame_num; + msg_meta->confidence = obj_meta->confidence; + msg_meta->trackingId = obj_meta->object_id; + msg_meta->ts = (gchar *) g_malloc0 (MAX_TIME_STAMP_LEN + 1); + generate_ts_rfc3339(msg_meta->ts, MAX_TIME_STAMP_LEN); + msg_meta->objectId = (gchar *) g_malloc0 (MAX_LABEL_SIZE); + strncpy(msg_meta->objectId, obj_meta->obj_label, MAX_LABEL_SIZE); + // msg_meta->sensorStr = + // msg_meta->otherAttr = + // msg_meta->videoPath = + // // DEBUG + // g_message("Metadata is built."); + + NvDsBatchMeta *batch_meta = frame_meta->base_meta.batch_meta; + NvDsUserMeta *user_event_meta = nvds_acquire_user_meta_from_pool (batch_meta); + if (user_event_meta) { + user_event_meta->user_meta_data = (void *) msg_meta; + user_event_meta->base_meta.meta_type = NVDS_EVENT_MSG_META; + user_event_meta->base_meta.copy_func = (NvDsMetaCopyFunc) copy_bodypose_meta; + user_event_meta->base_meta.release_func = (NvDsMetaReleaseFunc)release_bodypose_meta; + nvds_add_user_meta_to_frame(frame_meta, user_event_meta); + } else { + g_printerr("Error in attaching event meta to buffer\n"); + } +} + +void osd_upper_body(NvDsFrameMeta* frame_meta, + NvDsBatchMeta *bmeta, + NvDsDisplayMeta *dmeta, + const int numKeyPoints, + const float keypoints[], + const float keypoints_confidence[]) +{ + const int keypoint_radius = 3 * _image_width / MUXER_OUTPUT_WIDTH;//6;//3; + const int keypoint_line_width = 2 * _image_width / MUXER_OUTPUT_WIDTH;//4;//2; + + const int num_joints = 24; + const int idx_joints[] = { 0, 1, 2, 3, 6, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33}; + const int num_bones = 25; + const int idx_bones[] = { 21, 6, 20, 6, 21, 23, 20, 22, 24, 22, 23, 25, + 27, 25, 31, 25, 33, 25, 29, 25, 24, 30, 24, 26, + 24, 32, 24, 28, 2, 21, 1, 20, 3, 6, 6, 15, + 15, 16, 15, 17, 19, 17, 18, 16, 0, 1, 0, 2, + 0, 3}; + const NvOSD_ColorParams bone_colors[] = { + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}}; + + for (int ii = 0; ii < num_joints; ii++) { + int i = idx_joints[ii]; + + if (keypoints_confidence[i] < _sgie_classifier_threshold) + continue; + + ACQUIRE_DISP_META(dmeta); + NvOSD_CircleParams &cparams = dmeta->circle_params[dmeta->num_circles]; + cparams.xc = keypoints[2 * i ]; + cparams.yc = keypoints[2 * i + 1]; + cparams.radius = keypoint_radius; + cparams.circle_color = NvOSD_ColorParams{0.96, 0.26, 0.21, 1}; + cparams.has_bg_color = 1; + cparams.bg_color = NvOSD_ColorParams{0.96, 0.26, 0.21, 1}; + dmeta->num_circles++; + } + + for (int i = 0; i < num_bones; i++) { + int i0 = idx_bones[2 * i ]; + int i1 = idx_bones[2 * i + 1]; + + if ((keypoints_confidence[i0] < _sgie_classifier_threshold) || + (keypoints_confidence[i1] < _sgie_classifier_threshold)) + continue; + + ACQUIRE_DISP_META(dmeta); + NvOSD_LineParams *lparams = &dmeta->line_params[dmeta->num_lines]; + lparams->x1 = keypoints[2 * i0]; + lparams->y1 = keypoints[2 * i0 + 1]; + lparams->x2 = keypoints[2 * i1]; + lparams->y2 = keypoints[2 * i1 + 1]; + lparams->line_width = keypoint_line_width; + lparams->line_color = bone_colors[i]; + dmeta->num_lines++; + } + + return; +} + +void osd_lower_body(NvDsFrameMeta* frame_meta, + NvDsBatchMeta *bmeta, + NvDsDisplayMeta *dmeta, + const int numKeyPoints, + const float keypoints[], + const float keypoints_confidence[]) +{ + const int keypoint_radius = 3 * _image_width / MUXER_OUTPUT_WIDTH;//6;//3; + const int keypoint_line_width = 2 * _image_width / MUXER_OUTPUT_WIDTH;//4;//2; + + const int num_joints = 10; + const int idx_joints[] = { 4, 5, 7, 8, 9, 10, 11, 12, 13, 14}; + const int num_bones = 10; + const int idx_bones[] = { 2, 5, 5, 8, 1, 4, 4, 7, 7, 13, + 8, 14, 8, 10, 7, 9, 11, 9, 12, 10}; + const NvOSD_ColorParams bone_colors[] = { + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}}; + + for (int ii = 0; ii < num_joints; ii++) { + int i = idx_joints[ii]; + + if (keypoints_confidence[i] < _sgie_classifier_threshold) + continue; + + ACQUIRE_DISP_META(dmeta); + NvOSD_CircleParams &cparams = dmeta->circle_params[dmeta->num_circles]; + cparams.xc = keypoints[2 * i ]; + cparams.yc = keypoints[2 * i + 1]; + cparams.radius = keypoint_radius; + cparams.circle_color = NvOSD_ColorParams{0.96, 0.26, 0.21, 1}; + cparams.has_bg_color = 1; + cparams.bg_color = NvOSD_ColorParams{0.96, 0.26, 0.21, 1}; + dmeta->num_circles++; + } + + for (int i = 0; i < num_bones; i++) { + int i0 = idx_bones[2 * i ]; + int i1 = idx_bones[2 * i + 1]; + + if ((keypoints_confidence[i0] < _sgie_classifier_threshold) || + (keypoints_confidence[i1] < _sgie_classifier_threshold)) + continue; + + ACQUIRE_DISP_META(dmeta); + NvOSD_LineParams *lparams = &dmeta->line_params[dmeta->num_lines]; + lparams->x1 = keypoints[2 * i0]; + lparams->y1 = keypoints[2 * i0 + 1]; + lparams->x2 = keypoints[2 * i1]; + lparams->y2 = keypoints[2 * i1 + 1]; + lparams->line_width = keypoint_line_width; + lparams->line_color = bone_colors[i]; + dmeta->num_lines++; + } + + return; +} + +void parse_25dpose_from_tensor_meta(NvDsInferTensorMeta *tensor_meta, + NvDsFrameMeta *frame_meta, NvDsObjectMeta *obj_meta) +{ + // const int pelvis = 0; + // const int left_hip = 1; + // const int right_hip = 2; + // const int torso = 3; + // const int left_knee = 4; + // const int right_knee = 5; + // const int neck = 6; + // const int left_ankle = 7; + // const int right_ankle = 8; + // const int left_big_toe = 9; + // const int right_big_toe = 10; + // const int left_small_toe = 11; + // const int right_small_toe = 12; + // const int left_heel = 13; + // const int right_heel = 14; + // const int nose = 15; + // const int left_eye = 16; + // const int right_eye = 17; + // const int left_ear = 18; + // const int right_ear = 19; + // const int left_shoulder = 20; + // const int right_shoulder = 21; + // const int left_elbow = 22; + // const int right_elbow = 23; + // const int left_wrist = 24; + // const int right_wrist = 25; + // const int left_pinky_knuckle = 26; + // const int right_pinky_knuckle = 27; + // const int left_middle_tip = 28; + // const int right_middle_tip = 29; + // const int left_index_knuckle = 30; + // const int right_index_knuckle = 31; + // const int left_thumb_tip = 32; + // const int right_thumb_tip = 33; + + float threshold = 0.1; + int window_size = 5; + int max_num_parts = 20; + int num_integral_samples = 7; + float link_threshold = 0.1; + int max_num_objects = 100; + + void *cmap_data = tensor_meta->out_buf_ptrs_host[0]; + NvDsInferDims &cmap_dims = tensor_meta->output_layers_info[0].inferDims; + void *paf_data = tensor_meta->out_buf_ptrs_host[1]; + NvDsInferDims &paf_dims = tensor_meta->output_layers_info[1].inferDims; + + const int numKeyPoints = 34; + // std::vector keypoints(68) ; + // std::vector keypointsZRel(34); + // std::vector keypoints_confidence(34); + float keypoints[2 * numKeyPoints]; + float keypointsZRel[numKeyPoints]; + float keypoints_confidence[numKeyPoints]; + + m_K_inv_transpose = _K.inverse().eval(); + m_K_inv_transpose = m_K_inv_transpose.transpose().eval(); + + NvDsBatchMeta *bmeta = frame_meta->base_meta.batch_meta; + NvDsDisplayMeta *dmeta = nvds_acquire_display_meta_from_pool(bmeta); + nvds_add_display_meta_to_frame(frame_meta, dmeta); + + for (unsigned int m=0; m < tensor_meta->num_output_layers;m++){ + NvDsInferLayerInfo *info = &tensor_meta->output_layers_info[m]; + + if (!strcmp(info->layerName, "pose25d")) { + float *data = (float *)tensor_meta->out_buf_ptrs_host[m]; + /* for (int j =0 ; j < 34; j++){ + printf ("a=%f b=%f c=%f d=%f\n",data[j*4],data[j*4+1],data[j*4+2], data[j*4+3]); + }*/ + + // Initialize + if (g_filter_pose25d.find(obj_meta->object_id) == g_filter_pose25d.end()) { + const float m_oneEuroSampleRate = 30.0f; + // const float m_oneEuroMinCutoffFreq = 0.1f; + // const float m_oneEuroCutoffSlope = 0.05f; + const float m_oneEuroDerivCutoffFreq = 1.0f;// Hz + + //std::vector filter_vec; + std::vector filter_vec; + + for (int j=0; j < numKeyPoints*3; j++) { + //TODO:Pending delete especially when object goes out of view, or ID switch + //will cause memleak, cleanup required wrap into class + // filter_vec.push_back(SF1eFilterCreate(30, 1.0, 0.0, 1.0)); + + // filters for x and y + // for (auto& fil : m_filterKeypoints2D) fil.reset(m_oneEuroSampleRate, 0.1f, 0.05, m_oneEuroDerivCutoffFreq); + filter_vec.push_back(OneEuroFilter(m_oneEuroSampleRate, 0.1f, 0.05, m_oneEuroDerivCutoffFreq)); + filter_vec.push_back(OneEuroFilter(m_oneEuroSampleRate, 0.1f, 0.05, m_oneEuroDerivCutoffFreq)); + + // filters for z (depth) + // for (auto& fil : m_filterKeypointsRelDepth) fil.reset(m_oneEuroSampleRate, 0.5f, 0.05, m_oneEuroDerivCutoffFreq); + filter_vec.push_back(OneEuroFilter(m_oneEuroSampleRate, 0.5f, 0.05, m_oneEuroDerivCutoffFreq)); + } + g_filter_pose25d[obj_meta->object_id] = filter_vec; + + // Filters depth of root keypoint + m_filterRootDepth.reset(m_oneEuroSampleRate, 0.1f, 0.05f, m_oneEuroDerivCutoffFreq); + } + + int batchSize_offset = 0; + + //std::vector &filt_val = g_filter_pose25d[obj_meta->object_id]; + std::vector &filt_val = g_filter_pose25d[obj_meta->object_id]; + + // x,y,z,c + for (int i = 0; i < numKeyPoints; i++) { + int index = batchSize_offset + i * 4; + + // Update with filtered results + keypoints[2 * i ] = filt_val[3 * i ].filter(data[index ] * + (obj_meta->rect_params.width / 192.0) + obj_meta->rect_params.left); + keypoints[2 * i + 1] = filt_val[3 * i + 1].filter(data[index + 1] * + (obj_meta->rect_params.height / 256.0) + obj_meta->rect_params.top); + keypointsZRel[i] = filt_val[3 * i + 2].filter(data[index + 2]); + + keypoints_confidence[i] = data[index + 3]; + } + + // Since we have cropped and resized the image buffer provided to the SDK from the app, + // we scale and offset the points back to the original resolution + float scaleOffsetXY[] = {1.0f, 0.0f, 1.0f, 0.0f}; + + // Render upper body + if (1) { + osd_upper_body(frame_meta, bmeta, dmeta, numKeyPoints, keypoints, keypoints_confidence); + } + // Render lower body + if (1) { + osd_lower_body(frame_meta, bmeta, dmeta, numKeyPoints, keypoints, keypoints_confidence); + } + + // SGIE operates on an enlarged/padded image buffer. + // const int muxer_output_width_pad = _pad_dim * 2 + _image_width; + // const int muxer_output_height_pad = _pad_dim * 2 + _image_height; + // Before outputting result, the image frame with overlay is cropped by removing _pad_dim. + // The final pose estimation result should counter the padding before deriving 3D keypoints. + for (int i = 0; i < numKeyPoints; i++) { + keypoints[2 * i ]-= _pad_dim; + keypoints[2 * i + 1]-= _pad_dim; + } + + // Recover pose 3D + std::vector p3dLifted; + p3dLifted = liftKeypoints25DTo3D(keypoints, keypointsZRel, numKeyPoints, m_K_inv_transpose, m_scale_ll); + float scale = recoverScale(p3dLifted, keypoints_confidence, m_mean_ll); + // printf("scale = %f\n", scale); + for (auto i = 0; i < p3dLifted.size(); i++) { + p3dLifted[i].x *= scale; + p3dLifted[i].y *= scale; + p3dLifted[i].z *= scale; + } + + if (_nvmsgbroker_conn_str) {// Prepare metadata to message broker + build_msg_meta(frame_meta, obj_meta, + numKeyPoints, keypoints, keypointsZRel, + keypoints_confidence, p3dLifted); + g_debug("Sent metadata of frame %6d to message broker.", frame_meta->frame_num); + } + + // Output pose25d and pose3d tensors + if (_pose_file) { + fprintf(_pose_file, + "{\n" + " \"object_id\": %lu,\n", + obj_meta->object_id); + + // Write pose25d + fprintf(_pose_file, + " \"pose25d\": ["); + for (int i = 0; i < p3dLifted.size(); i++) { + // Remember the position of "," so that we can remove it on the last entry. + fprintf(_pose_file, "%f, %f, %f, %f", keypoints[2*i], keypoints[2*i+1], + keypointsZRel[i], keypoints_confidence[i]); + fgetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, ", "); + } + fsetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, "],\n"); + + // Write the recovered pose3d + fprintf(_pose_file, + " \"pose3d\": ["); + for (int i = 0; i < p3dLifted.size(); i++) { + // Remember the position of "," so that we can remove it on the last entry. + fprintf(_pose_file, "%f, %f, %f, %f", p3dLifted[i].x, p3dLifted[i].y, p3dLifted[i].z, keypoints_confidence[i]); + fgetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, ", "); + } + fsetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, "]\n"); + + // Remember the position of "," so that we can remove it on the last entry. + fprintf(_pose_file, " }"); + fgetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, ", "); + } + } + } +} + +/* pgie_src_pad_buffer_probe will extract metadata received from pgie + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +pgie_src_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) +{ + gchar *msg = NULL; + GstBuffer *buf = (GstBuffer *)info->data; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsMetaList *l_user = NULL; + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + /* Padding due to AR SDK model requires bigger bboxes*/ + const int muxer_output_width_pad = _pad_dim * 2 + _image_width; + const int muxer_output_height_pad = _pad_dim * 2 + _image_height; + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) + { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)l_obj->data; + float sizex = obj_meta->rect_params.width * .5f; + float sizey = obj_meta->rect_params.height * .5f; + float centrx = obj_meta->rect_params.left + sizex; + float centry = obj_meta->rect_params.top + sizey; + sizex *= (1.25f); + sizey *= (1.25f); + if (sizex < sizey) + sizex = sizey; + else + sizey = sizex; + + obj_meta->rect_params.width = roundf(2.f *sizex); + obj_meta->rect_params.height = roundf(2.f *sizey); + obj_meta->rect_params.left = roundf (centrx - obj_meta->rect_params.width/2.f); + obj_meta->rect_params.top = roundf (centry - obj_meta->rect_params.height/2.f); + + sizex= obj_meta->rect_params.width * .5f, sizey = obj_meta->rect_params.height * .5f; + centrx = obj_meta->rect_params.left + sizex, centry = obj_meta->rect_params.top + sizey; + // Make sure box has same aspect ratio as 3D Body Pose model's input dimensions + // (e.g 192x256 -> 0.75 aspect ratio) by enlarging in the appropriate dimension. + float xScale = (float)192.0 / (float)sizex, yScale = (float)256.0 / (float)sizey; + if (xScale < yScale) { // expand on height + sizey = (float)256.0/ xScale; + } + else { // expand on width + sizex = (float)192.0 / yScale; + } + + obj_meta->rect_params.width = roundf(2.f *sizex); + obj_meta->rect_params.height = roundf(2.f *sizey); + obj_meta->rect_params.left = roundf (centrx - obj_meta->rect_params.width/2.f); + obj_meta->rect_params.top = roundf (centry - obj_meta->rect_params.height/2.f); + if (obj_meta->rect_params.left < 0.0) { + obj_meta->rect_params.left = 0.0; + } + if (obj_meta->rect_params.top < 0.0) { + obj_meta->rect_params.top = 0.0; + } + if (obj_meta->rect_params.left + obj_meta->rect_params.width > muxer_output_width_pad -1){ + obj_meta->rect_params.width = muxer_output_width_pad - 1 - obj_meta->rect_params.left; + } + if (obj_meta->rect_params.top + obj_meta->rect_params.height > muxer_output_height_pad -1){ + obj_meta->rect_params.height = muxer_output_height_pad - 1 - obj_meta->rect_params.top; + } + + } + } + return GST_PAD_PROBE_OK; +} + +/* sgie_src_pad_buffer_probe will extract metadata received from pgie + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +sgie_src_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) +{ + gchar *msg = NULL; + GstBuffer *buf = (GstBuffer *)info->data; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsMetaList *l_user = NULL; + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + if (_pose_file) {// Write batch header + if (batch_meta->frame_meta_list) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(batch_meta->frame_meta_list->data); + if (frame_meta->obj_meta_list) { + fprintf(_pose_file, + "{\n" + " \"num_frames_in_batch\": %d,\n" + " \"batches\": [", + batch_meta->num_frames_in_batch); + } + } + } + + // g_mutex_lock(&str->struct_lock); + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + + if (_pose_file) {// Write frame header + if (frame_meta->obj_meta_list) { + fprintf(_pose_file, + "{\n" + " \"batch_id\": %d,\n" + " \"frame_num\": %d,\n" + " \"ntp_timestamp\": %ld,\n" + " \"num_obj_meta\": %d,\n" + " \"objects\": [", + frame_meta->batch_id, frame_meta->frame_num, frame_meta->ntp_timestamp, frame_meta->num_obj_meta); + fgetpos(_pose_file, &g_fp_25_pos); + } + } + + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) + { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)l_obj->data; + + // Set below values to 0 in order to disable bbox and text output + obj_meta->rect_params.border_width = 0;//2; + obj_meta->text_params.font_params.font_size = 0;//10; + + for (l_user = obj_meta->obj_user_meta_list; l_user != NULL; + l_user = l_user->next) + { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type == NVDSINFER_TENSOR_OUTPUT_META) + { + NvDsInferTensorMeta *tensor_meta = + (NvDsInferTensorMeta *)user_meta->user_meta_data; + parse_25dpose_from_tensor_meta(tensor_meta, frame_meta, obj_meta) ; + } + } + } + + if (_pose_file) {// closing off "objects" key. + if (frame_meta->obj_meta_list) { + fsetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, "]\n");// closing off "objects" key. + + fprintf(_pose_file, " }"); + fgetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, ", "); + } + } + } + + if (_pose_file) {// closing off "batches" key. + if (batch_meta->frame_meta_list) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(batch_meta->frame_meta_list->data); + if (frame_meta->obj_meta_list) { + fsetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, "]\n");// closing off "batches" key. + + fprintf(_pose_file, "}"); + fgetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, ", "); + } + } + } + // g_mutex_unlock (&str->struct_lock); + + return GST_PAD_PROBE_OK; +} + +/* osd_sink_pad_buffer_probe will extract metadata received from OSD + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +osd_sink_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) +{ + GstBuffer *buf = (GstBuffer *)info->data; + guint num_rects = 0; + NvDsObjectMeta *obj_meta = NULL; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsDisplayMeta *display_meta = NULL; + + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + int offset = 0; + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) + { + obj_meta = (NvDsObjectMeta *)(l_obj->data); + } + display_meta = nvds_acquire_display_meta_from_pool(batch_meta); + + /* Parameters to draw text onto the On-Screen-Display */ + NvOSD_TextParams *txt_params = &display_meta->text_params[0]; + display_meta->num_labels = 1; + txt_params->display_text = (char *)g_malloc0(MAX_DISPLAY_LEN); + offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "Frame Number = %d", frame_number); + offset = snprintf(txt_params->display_text + offset, MAX_DISPLAY_LEN, " "); + + txt_params->x_offset = 10; + txt_params->y_offset = 12; + + char font_name[] = "Mono"; + txt_params->font_params.font_name = font_name; + txt_params->font_params.font_size = 10; + txt_params->font_params.font_color.red = 1.0; + txt_params->font_params.font_color.green = 1.0; + txt_params->font_params.font_color.blue = 1.0; + txt_params->font_params.font_color.alpha = 1.0; + + txt_params->set_bg_clr = 1; + txt_params->text_bg_clr.red = 0.0; + txt_params->text_bg_clr.green = 0.0; + txt_params->text_bg_clr.blue = 0.0; + txt_params->text_bg_clr.alpha = 1.0; + + nvds_add_display_meta_to_frame(frame_meta, display_meta); + } + frame_number++; + return GST_PAD_PROBE_OK; +} + +static gboolean +bus_call(GstBus *bus, GstMessage *msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *)data; + switch (GST_MESSAGE_TYPE(msg)) + { + case GST_MESSAGE_EOS: + g_print("End of Stream\n"); + g_main_loop_quit(loop); + break; + + case GST_MESSAGE_ERROR: + { + gchar *debug; + GError *error; + gst_message_parse_error(msg, &error, &debug); + g_printerr("ERROR from element %s: %s\n", + GST_OBJECT_NAME(msg->src), error->message); + if (debug) + g_printerr("Error details: %s\n", debug); + g_free(debug); + g_error_free(error); + g_main_loop_quit(loop); + break; + } + + default: + break; + } + return TRUE; +} + +gboolean +link_element_to_tee_src_pad(GstElement *tee, GstElement *sinkelem) +{ + gboolean ret = FALSE; + GstPad *tee_src_pad = NULL; + GstPad *sinkpad = NULL; + GstPadTemplate *padtemplate = NULL; + + padtemplate = (GstPadTemplate *)gst_element_class_get_pad_template(GST_ELEMENT_GET_CLASS(tee), "src_%u"); + tee_src_pad = gst_element_request_pad(tee, padtemplate, NULL, NULL); + + if (!tee_src_pad) + { + g_printerr("Failed to get src pad from tee"); + goto done; + } + + sinkpad = gst_element_get_static_pad(sinkelem, "sink"); + if (!sinkpad) + { + g_printerr("Failed to get sink pad from '%s'", + GST_ELEMENT_NAME(sinkelem)); + goto done; + } + + if (gst_pad_link(tee_src_pad, sinkpad) != GST_PAD_LINK_OK) + { + g_printerr("Failed to link '%s' and '%s'", GST_ELEMENT_NAME(tee), + GST_ELEMENT_NAME(sinkelem)); + goto done; + } + ret = TRUE; + +done: + if (tee_src_pad) + { + gst_object_unref(tee_src_pad); + } + if (sinkpad) + { + gst_object_unref(sinkpad); + } + return ret; +} + +static void +cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data) +{ + g_print ("In cb_newpad\n"); + GstCaps *caps = gst_pad_get_current_caps (decoder_src_pad); + const GstStructure *str = gst_caps_get_structure (caps, 0); + const gchar *name = gst_structure_get_name (str); + GstElement *source_bin = (GstElement *) data; + GstCapsFeatures *features = gst_caps_get_features (caps, 0); + + /* Need to check if the pad created by the decodebin is for video and not + * audio. */ + if (!strncmp (name, "video", 5)) { + /* Link the decodebin pad only if decodebin has picked nvidia + * decoder plugin nvdec_*. We do this by checking if the pad caps contain + * NVMM memory features. */ + if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) { + /* Get the source bin ghost pad */ + GstPad *bin_ghost_pad = gst_element_get_static_pad (source_bin, "src"); + if (!gst_ghost_pad_set_target (GST_GHOST_PAD (bin_ghost_pad), + decoder_src_pad)) { + g_printerr ("Failed to link decoder src pad to source bin ghost pad\n"); + } + gst_object_unref (bin_ghost_pad); + } else { + g_printerr ("Error: Decodebin did not pick nvidia decoder plugin.\n"); + } + } +} + +static void +decodebin_child_added (GstChildProxy * child_proxy, GObject * object, + gchar * name, gpointer user_data) +{ + g_print ("Decodebin child added: %s\n", name); + if (g_strrstr (name, "urisourcebin") == name) { + g_signal_connect (G_OBJECT (object), "child-added", + G_CALLBACK (decodebin_child_added), user_data); + } + /* Commented to suppress warnings*/ + /*if (!(g_strcmp0 (source_type, "rtmp"))) { + g_object_set (G_OBJECT (object), "do-timestamp", 1, NULL); + g_object_set (G_OBJECT (object), "timeout", 10000, NULL); + }*/ +} + +// Imported from deepstream_test3_app.c +static GstElement * +create_source_bin (guint index, gchar * uri) +{ + GstElement *bin = NULL, *uri_decode_bin = NULL; + gchar bin_name[16] = { }; + + g_snprintf (bin_name, 15, "source-bin-%02d", index); + /* Create a source GstBin to abstract this bin's content from the rest of the + * pipeline */ + bin = gst_bin_new (bin_name); + + /* Source element for reading from the uri. + * We will use decodebin and let it figure out the container format of the + * stream and the codec and plug the appropriate demux and decode plugins. */ + uri_decode_bin = gst_element_factory_make ("uridecodebin", "uri-decode-bin"); + + if (!bin || !uri_decode_bin) { + g_printerr ("One element in source bin could not be created.\n"); + return NULL; + } + + /* We set the input uri to the source element */ + g_object_set (G_OBJECT (uri_decode_bin), "uri", uri, NULL); + + /* Connect to the "pad-added" signal of the decodebin which generates a + * callback once a new pad for raw data has beed created by the decodebin */ + g_signal_connect (G_OBJECT (uri_decode_bin), "pad-added", + G_CALLBACK (cb_newpad), bin); + g_signal_connect (G_OBJECT (uri_decode_bin), "child-added", + G_CALLBACK (decodebin_child_added), bin); + + gst_bin_add (GST_BIN (bin), uri_decode_bin); + + /* We need to create a ghost pad for the source bin which will act as a proxy + * for the video decoder src pad. The ghost pad will not have a target right + * now. Once the decode bin creates the video decoder and generates the + * cb_newpad callback, we will set the ghost pad target to the video decoder + * src pad. */ + if (!gst_element_add_pad (bin, gst_ghost_pad_new_no_target ("src", + GST_PAD_SRC))) { + g_printerr ("Failed to add ghost pad in source bin\n"); + return NULL; + } + + return bin; +} + + +/** + * Function to handle program interrupt signal. + * It installs default handler after handling the interrupt. + */ +static void +_intr_handler (int signum) +{ + struct sigaction action; + + NVGSTDS_ERR_MSG_V ("User Interrupted.. \n"); + + memset (&action, 0, sizeof (action)); + action.sa_handler = SIG_DFL; + + sigaction (SIGINT, &action, NULL); + + _cintr = TRUE; +} + +/* + * Function to install custom handler for program interrupt signal. + */ +static void +_intr_setup (void) +{ + struct sigaction action; + + memset (&action, 0, sizeof (action)); + action.sa_handler = _intr_handler; + + sigaction (SIGINT, &action, NULL); +} + +/** + * Loop function to check the status of interrupts. + * It comes out of loop if application got interrupted. + */ +static gboolean +check_for_interrupt (gpointer data) +{ + if (_quit) { + return FALSE; + } + + if (_cintr) { + _cintr = FALSE; + + _quit = TRUE; + GMainLoop *loop = (GMainLoop *) data; + g_main_loop_quit (loop); + + return FALSE; + } + return TRUE; +} +//===from deepstream_test5_app_main.c=== + +bool verify_arguments() +{ + if (!_input) { + g_printerr("--input option is not specified. Exiting...\n"); + return false; + } + else { + if (strncmp(_input, "rtsp://", 7) && strncmp(_input, "file://", 7)) { + g_printerr("--input value is not a valid URI address. Exiting...\n"); + return false; + } + } + + if (_tracker) { + if (strcmp(_tracker, "accuracy") && strcmp(_tracker, "perf")) { + g_printerr("--tracker value is neither \"accuracy\", nor \"perf\". Exiting...\n"); + return false; + } + } + else {// default value + _tracker = (gchar *)g_malloc0(64); + strcpy(_tracker, "perf"); + } + + if (_pose_filename) { + _pose_file = fopen(_pose_filename, "wt"); + if (!_pose_file) { + g_printerr("Cannot open file %s. Exiting...\n", _pose_filename); + return false; + } + fprintf(_pose_file, "["); + fgetpos(_pose_file, &g_fp_25_pos); + } + + if (_publish_pose) { + if (strcmp(_publish_pose, "pose3d") && strcmp(_publish_pose, "pose25d")) { + g_printerr("--publish-pose value is neither \"pose3d\", nor \"pose25d\". Exiting...\n"); + return false; + } + } + + if (_image_width <= 0) { + g_printerr("--width value %d is non-positive. Exiting...\n", _image_width); + return false; + } + if (_image_height <= 0) { + g_printerr("--height value %d is non-positive. Exiting...\n", _image_height); + return false; + } + _focal_length = (float)_focal_length_dbl; + if (_focal_length <= 0) { + g_printerr("--focal value %f is non-positive. Exiting...\n", _focal_length); + return false; + } + + _K.row(0) << _focal_length, 0, _image_width / 2.f; + _K.row(1) << 0, _focal_length, _image_height / 2.f; + _K.row(2) << 0, 0, 1.f; + + _pad_dim = PAD_DIM * _image_width / MUXER_OUTPUT_WIDTH; + + return true; +} + +int main(int argc, char *argv[]) +{ + const guint num_sources = 1; + + GMainLoop *loop = NULL; + GstCaps *caps = NULL; + GstElement *source = NULL, *streammux_pgie = NULL; + GstElement *sink = NULL, *pgie = NULL; + // Padding the image and removing the padding + GstElement *nvvideoconvert_enlarge = NULL, *nvvideoconvert_reduce = NULL, + *capsFilter_enlarge = NULL, *capsFilter_reduce = NULL; + GstElement *nvvidconv = NULL, *nvosd = NULL, *tracker = NULL, *nvdslogger = NULL, + *filesink = NULL, *nvvideoencfilesinkbin = NULL, + *nvrtspoutsinkbin = NULL; + GstElement *tee = NULL, *msgbroker = NULL, *msgconv = NULL;// msg broker and converter + GstBus *bus = NULL; + guint bus_watch_id; + + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + //---Parse command line options--- + { + const GOptionEntry entries[] = { + {"version", 'v', 0, G_OPTION_ARG_NONE, &_print_version, + "Print DeepStreamSDK version.", NULL} + , + {"version-all", 0, 0, G_OPTION_ARG_NONE, &_print_dependencies_version, + "Print DeepStreamSDK and dependencies version.", NULL} + , + {"input", 0, 0, G_OPTION_ARG_STRING, &_input, + "[Required] Input video address in URI format by starting \ +with \"rtsp://\" or \"file://\".", + NULL} + , + {"output", 0, 0, G_OPTION_ARG_STRING, &_output, + "Output video address. Either \"rtsp://\" or a file path or \"fakesink\" is \ +acceptable. If the value is \"rtsp://\", then the result video is \ +published at \"rtsp://localhost:8554/ds-test\".", + NULL} + , + {"save-pose", 0, 0, G_OPTION_ARG_STRING, &_pose_filename, + "The file path to save both the pose25d and the recovered \ +pose3d in JSON format.", + NULL} + , + {"conn-str", 0, 0, G_OPTION_ARG_STRING, &_nvmsgbroker_conn_str, + "Connection string for Gst-nvmsgbroker, e.g. ;;.", + NULL} + , + {"publish-pose", 0, 0, G_OPTION_ARG_STRING, &_publish_pose, + "Specify the type of pose to publish. Acceptable \ +value is either \"pose3d\" or \"pose25d\". If not specified, both \"pose3d\" and \"pose25d\" \ +are published to the message broker.", + NULL} + , + {"tracker", 0, 0, G_OPTION_ARG_STRING, &_tracker, + "Specify the NvDCF tracker mode. The acceptable value is either \ +\"accuracy\" or \"perf\". The default value is \"perf\" \"accuracy\" mode"\ +" requires DeepSORT model to be installed. Please refer to [Setup Official Re-ID Model]"\ +"(https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvtracker.html) section for details.", + NULL} + , + {"fps", 0, 0, G_OPTION_ARG_NONE, &_print_fps, + "Print FPS in the format of current_fps (averaged_fps).", + NULL} + , + {"fps-interval", 0, 0, G_OPTION_ARG_INT, &_fps_interval, + "Interval in seconds to print the fps, applicable only with --fps flag.", + NULL} + , + {"width", 0, 0, G_OPTION_ARG_INT, &_image_width, + "Input video width in pixels. The default value is 1280.",//MUXER_OUTPUT_WIDTH + NULL} + , + {"height", 0, 0, G_OPTION_ARG_INT, &_image_height, + "Input video height in pixels. The default value is 720.",//MUXER_OUTPUT_HEIGHT + NULL} + , + {"focal", 0, 0, G_OPTION_ARG_DOUBLE, &_focal_length_dbl, + "Camera focal length in millimeters. The default value is 800.79041.",//FOCAL_LENGTH + NULL} + , + {"osd-process-mode", 0, 0, G_OPTION_ARG_INT, &_osd_process_mode, + "OSD process mode CPU - 0 or GPU 1.", + NULL} + , + {NULL} + , + }; + + GOptionContext *ctx = NULL; + GOptionGroup *group = NULL; + GError *error = NULL; + guint i; + + ctx = g_option_context_new ("Deepstream BodyPose3DNet App"); + group = g_option_group_new ("arguments", NULL, NULL, NULL, NULL); + g_option_group_add_entries (group, entries); + + g_option_context_set_main_group (ctx, group); + g_option_context_add_group (ctx, gst_init_get_option_group ()); + + GST_DEBUG_CATEGORY_INIT (NVDS_APP, "Deepstream BodyPose3DNet App", 0, NULL); + + if (!g_option_context_parse (ctx, &argc, &argv, &error)) { + NVGSTDS_ERR_MSG_V ("%s", error->message); + g_printerr ("%s",g_option_context_get_help (ctx, TRUE, NULL)); + return -1; + } + + if (_print_version) { + g_print ("deepstream-bodypose3dnet-app version %d.%d.%d\n", + NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO); + return 0; + } + + if (_print_dependencies_version) { + g_print ("deepstream-bodypose3dnet-app version %d.%d.%d\n", + NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO); + return 0; + } + + if (!verify_arguments()) { + g_printerr ("%s",g_option_context_get_help (ctx, TRUE, NULL)); + return -1; + } + } + //---Parse command line options--- + + + /* Standard GStreamer initialization */ + // signal(SIGINT, sigintHandler); + gst_init(&argc, &argv); + loop = g_main_loop_new(NULL, FALSE); + + _intr_setup (); + g_timeout_add (400, check_for_interrupt, loop); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + pipeline = gst_pipeline_new("deepstream-bodypose3dnet"); + if (!pipeline) { + g_printerr ("Pipeline could not be created. Exiting.\n"); + return -1; + } + + /* we add a message handler */ + bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline)); + bus_watch_id = gst_bus_add_watch(bus, bus_call, loop); + gst_object_unref(bus); + + /* Create nvstreammux instance to form batches from one or more sources. */ + streammux_pgie = gst_element_factory_make ("nvstreammux", "streammux-pgie"); + if (!streammux_pgie) { + g_printerr ("PGIE streammux could not be created. Exiting.\n"); + return -1; + } + //---Set properties of streammux_pgie--- + g_object_set(G_OBJECT(streammux_pgie), "batch-size", num_sources, NULL); + g_object_set(G_OBJECT(streammux_pgie), "width", _image_width, "height", + _image_height, + "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); + + gst_bin_add(GST_BIN(pipeline), streammux_pgie); + //---Set properties of streammux_pgie--- + + // !!!TODO: support >1 input streams!!! + /* Source element for reading from the file/uri */ + { + GstPad *sinkpad, *srcpad; + gchar pad_name[16] = { }; + + source = create_source_bin(0, const_cast(_input)); + if (!source) { + g_printerr ("Failed to create source bin. Exiting.\n"); + return -1; + } + gst_bin_add(GST_BIN(pipeline), source); + + g_snprintf (pad_name, 15, "sink_%u", 0); + sinkpad = gst_element_get_request_pad(streammux_pgie, pad_name); + if (!sinkpad) { + g_printerr ("Source Streammux request sink pad failed. Exiting.\n"); + return -1; + } + + srcpad = gst_element_get_static_pad(source, "src"); + if (!srcpad) { + g_printerr ("Failed to get src pad of source bin. Exiting.\n"); + return -1; + } + + if (gst_pad_link(srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Failed to link source bin to stream muxer. Exiting.\n"); + return -1; + } + + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + } + + /* Use nvinfer to run inferencing on decoder's output, + * behaviour of inferencing is set through config file */ + pgie = gst_element_factory_make("nvinfer", "primary-nvinference-engine"); + if (!pgie) { + g_printerr ("PGIE element could not be created. Exiting.\n"); + return -1; + } + //---Set pgie properties--- + /* Configure the nvinfer element using the nvinfer config file. */ + g_object_set(G_OBJECT(pgie), "config-file-path", PGIE_CONFIG_FILE, NULL); + + /* Override the batch-size set in the config file with the number of sources. */ + guint pgie_batch_size = 0; + g_object_get(G_OBJECT(pgie), "batch-size", &pgie_batch_size, NULL); + if (pgie_batch_size != num_sources) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + pgie_batch_size, num_sources); + + g_object_set(G_OBJECT(pgie), "batch-size", num_sources, NULL); + } + + //---Set pgie properties--- + + /* We need to have a tracker to track the identified objects */ + tracker = gst_element_factory_make ("nvtracker", "tracker"); + if (!tracker) { + g_printerr ("Nvtracker could not be created. Exiting.\n"); + return -1; + } + g_object_set (G_OBJECT(tracker), "ll-lib-file", + "/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so", + NULL); + + if (!strcmp(_tracker, "accuracy")) { + g_object_set(G_OBJECT(tracker), "ll-config-file", + "/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml", + NULL); + } + else if (!strcmp(_tracker, "perf")) { + g_object_set(G_OBJECT(tracker), "ll-config-file", + "/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml", + NULL); + } + + if (_print_fps){ + nvdslogger = gst_element_factory_make ("nvdslogger", "nvdslogger"); + if (!nvdslogger) { + g_printerr ("Nvdslogger could not be created. Exiting.\n"); + return -1; + } + if (_fps_interval){ + g_object_set (G_OBJECT(nvdslogger), "fps-measurement-interval-sec", + _fps_interval, + NULL); + } + } + else { + nvdslogger = gst_element_factory_make ("queue", NULL); + if (!nvdslogger) { + g_printerr ("queue could not be created. Exiting.\n"); + return -1; + } + } + + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the osd element, since by that time, the buffer would have + * had got all the metadata. */ + GstPad* pgie_src_pad = gst_element_get_static_pad(tracker, "src"); + if (!pgie_src_pad) + g_printerr ("Unable to get src pad for pgie\n"); + else + gst_pad_add_probe(pgie_src_pad, GST_PAD_PROBE_TYPE_BUFFER, + pgie_src_pad_buffer_probe, NULL, NULL); + gst_object_unref (pgie_src_pad); + + /* 3d bodypose secondary gie */ + GstElement* sgie = gst_element_factory_make("nvinfer", "secondary-nvinference-engine"); + if (!sgie) { + g_printerr ("Secondary nvinfer could not be created. Exiting.\n"); + return -1; + } + //---Set sgie properties--- + /* Configure the nvinfer element using the nvinfer config file. */ + g_object_set(G_OBJECT(sgie), + "output-tensor-meta", TRUE, + "config-file-path", SGIE_CONFIG_FILE, + NULL); + + /* Override the batch-size set in the config file with the number of sources. */ + guint sgie_batch_size = 0; + g_object_get(G_OBJECT(sgie), "batch-size", &sgie_batch_size, NULL); + if (sgie_batch_size < num_sources) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + sgie_batch_size, num_sources); + + g_object_set(G_OBJECT(sgie), "batch-size", num_sources, NULL); + } + + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the osd element, since by that time, the buffer would have + * had got all the metadata. */ + GstPad* sgie_src_pad = gst_element_get_static_pad(sgie, "src"); + if (!sgie_src_pad) + g_printerr("Unable to get src pad for sgie\n"); + else + gst_pad_add_probe(sgie_src_pad, GST_PAD_PROBE_TYPE_BUFFER, + sgie_src_pad_buffer_probe, NULL, NULL); + gst_object_unref(sgie_src_pad); + //---Set sgie properties--- + + /* Create tee to render buffer and send message simultaneously*/ + tee = gst_element_factory_make ("tee", "nvsink-tee"); + + /* Add queue elements between every two elements */ + GstElement* queue_nvvidconv = NULL; + queue_nvvidconv = gst_element_factory_make("queue", "queue_nvvidconv"); + if (!queue_nvvidconv) { + g_printerr ("queue_nvvidconv could not be created. Exiting.\n"); + return -1; + } + + /* Use convertor to convert from NV12 to RGBA as required by nvosd */ + nvvidconv = gst_element_factory_make("nvvideoconvert", "nvvideo-converter"); + if (!nvvidconv) { + g_printerr ("nvvidconv could not be created. Exiting.\n"); + return -1; + } + + //---Manipulate image size so that PGIE bbox is large enough--- + // Enlarge image so that PeopleNet detected bbox is larger which would fully cover the + // detected object in the original sized image. + nvvideoconvert_enlarge = gst_element_factory_make("nvvideoconvert", "nvvideoconvert_enlarge"); + if (!nvvideoconvert_enlarge) { + g_printerr ("nvvideoconvert_enlarge could not be created. Exiting.\n"); + return -1; + } + capsFilter_enlarge = gst_element_factory_make("capsfilter", "capsFilter_enlarge"); + if (!capsFilter_enlarge) { + g_printerr ("capsFilter_enlarge could not be created. Exiting.\n"); + return -1; + } + + // Reduce the previously enlarged image frame so that the final output video retains the + // same dimension as the pipeline's input video dimension. + nvvideoconvert_reduce = gst_element_factory_make("nvvideoconvert", "nvvideoconvert_reduce"); + if (!nvvideoconvert_reduce) { + g_printerr ("nvvideoconvert_reduce could not be created. Exiting.\n"); + return -1; + } + capsFilter_reduce = gst_element_factory_make("capsfilter", "capsFilter_reduce"); + if (!capsFilter_reduce) { + g_printerr ("capsFilter_reduce could not be created. Exiting.\n"); + return -1; + } + + gchar *string1 = NULL; + asprintf (&string1, "%d:%d:%d:%d", _pad_dim, _pad_dim, _image_width, _image_height); + // "dest-crop" - input size < output size + g_object_set(G_OBJECT(nvvideoconvert_enlarge), "dest-crop", string1,"interpolation-method",1 ,NULL); + // "src-crop" - input size > output size + g_object_set(G_OBJECT(nvvideoconvert_reduce), "src-crop", string1,"interpolation-method",1 ,NULL); + free(string1); + + /* Padding due to AR SDK model requires bigger bboxes*/ + const int muxer_output_width_pad = _pad_dim * 2 + _image_width; + const int muxer_output_height_pad = _pad_dim * 2 + _image_height; + asprintf (&string1, "video/x-raw(memory:NVMM),width=%d,height=%d", + muxer_output_width_pad, muxer_output_height_pad); + GstCaps *caps1 = gst_caps_from_string (string1); + g_object_set(G_OBJECT(capsFilter_enlarge),"caps", caps1, NULL); + free(string1); + gst_caps_unref(caps1); + + asprintf (&string1, "video/x-raw(memory:NVMM),width=%d,height=%d", + _image_width, _image_height); + caps1 = gst_caps_from_string (string1); + g_object_set(G_OBJECT(capsFilter_reduce),"caps", caps1, NULL); + free(string1); + gst_caps_unref(caps1); + //---Manipulate image size so that PGIE bbox is large enough--- + + /* Create OSD to draw on the converted RGBA buffer */ + nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); + if (!nvosd) { + g_printerr ("Nvdsosd could not be created. Exiting.\n"); + return -1; + } + + g_object_set (G_OBJECT(nvosd), "process-mode", _osd_process_mode, NULL); + + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the osd element, since by that time, the buffer would have + * had got all the metadata. */ + GstPad* osd_sink_pad = gst_element_get_static_pad(nvosd, "sink"); + if (!osd_sink_pad) + g_print("Unable to get sink pad\n"); + else + gst_pad_add_probe(osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + osd_sink_pad_buffer_probe, (gpointer)sink, NULL); + gst_object_unref(osd_sink_pad); + + /* Set output file location */ + if (_output) { + if (!strcmp(_output, "rtsp://")) { + filesink = gst_element_factory_make("nvrtspoutsinkbin", "nv-filesink"); + } + else if (!strcmp(_output,"fakesink")){ + filesink = gst_element_factory_make("fakesink", "nv-sink"); + } + else { + filesink = gst_element_factory_make("nvvideoencfilesinkbin", "nv-filesink"); + } + if (!filesink) { + g_printerr ("Filesink could not be created. Exiting.\n"); + return -1; + } + + if (strcmp(_output,"fakesink")){ + g_object_set(G_OBJECT(filesink), "output-file", _output, NULL); + g_object_set(G_OBJECT(filesink), "bitrate", 4000000, NULL); + //g_object_set(G_OBJECT(filesink), "profile", 3, NULL); + g_object_set(G_OBJECT(filesink), "codec", 2, NULL);//hevc + // g_object_set(G_OBJECT(filesink), "control-rate", 0, NULL);//hevc + } + } + else { +#ifdef __aarch64__ + filesink = gst_element_factory_make("nv3dsink", "nv3d-sink"); +#else + filesink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer"); +#endif + } + + /* Add all elements to the pipeline */ + // streammux_pgie has been added into pipeline already. + gst_bin_add_many(GST_BIN(pipeline), + nvvideoconvert_enlarge, capsFilter_enlarge, + pgie, tracker, sgie, tee, + queue_nvvidconv, nvvidconv, nvosd, filesink, nvdslogger, + nvvideoconvert_reduce, capsFilter_reduce, NULL); + + // Link elements + if (!gst_element_link_many(streammux_pgie, + nvvideoconvert_enlarge, capsFilter_enlarge, + pgie, tracker, sgie, nvdslogger, tee, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + if (prop.integrated && _output) { // Jetson + if (!gst_element_link_many(queue_nvvidconv, nvvidconv, nvosd, + nvvideoconvert_reduce, capsFilter_reduce, + filesink, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + } + else { // dGPU & Jetson Display + if (!gst_element_link_many(queue_nvvidconv, nvvidconv, nvosd, + nvvideoconvert_reduce, capsFilter_reduce, + filesink, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + } + + // Link tee and queue_nvvidconv + { + GstPad *sinkpad, *srcpad; + + srcpad = gst_element_get_request_pad (tee, "src_%u"); + sinkpad = gst_element_get_static_pad (queue_nvvidconv, "sink"); + if (!srcpad || !sinkpad) { + g_printerr ("Unable to get request pads\n"); + return -1; + } + + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Unable to link tee and queue_nvvidconv.\n"); + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + return -1; + } + + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + } + + if (_nvmsgbroker_conn_str) {// Publish metadata to a broker as well + GstElement* queue_msgconv = gst_element_factory_make("queue", "queue_msgconv"); + if (!queue_msgconv) { + g_printerr ("queue_msgconv could not be created. Exiting.\n"); + return -1; + } + + /* Set up message broker */ + /* Create msg converter to generate payload from buffer metadata */ + msgconv = gst_element_factory_make ("nvmsgconv", "nvmsg-converter"); + // g_object_set (G_OBJECT(msgconv), "config", MSGCONV_CONFIG_FILE, NULL); + g_object_set (G_OBJECT(msgconv), "payload-type", 1, NULL);// Minimal schema + g_object_set (G_OBJECT(msgconv), "msg2p-newapi", 0, NULL);// Event Msg meta + + /* Create msg broker to send payload to server */ + msgbroker = gst_element_factory_make ("nvmsgbroker", "nvmsg-broker"); + g_object_set (G_OBJECT(msgbroker), + "proto-lib", "/opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so", + "conn-str", _nvmsgbroker_conn_str, + "sync", FALSE, + NULL); + + gst_bin_add_many(GST_BIN(pipeline), queue_msgconv, msgconv, msgbroker, NULL); + + if (!gst_element_link_many(queue_msgconv, msgconv, msgbroker, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + + // Link tee with queue_msgconv + GstPad *sinkpad, *srcpad; + + srcpad = gst_element_get_request_pad (tee, "src_%u"); + sinkpad = gst_element_get_static_pad (queue_msgconv, "sink"); + if (!srcpad || !sinkpad) { + g_printerr ("Unable to get request pads\n"); + return -1; + } + + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Unable to link tee and queue_msgconv.\n"); + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + return -1; + } + + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + } + + /* Set the pipeline to "playing" state */ + g_print("Now playing: %s\n", _input); + gst_element_set_state(pipeline, GST_STATE_PLAYING); + GST_DEBUG_BIN_TO_DOT_FILE((GstBin*)pipeline, GST_DEBUG_GRAPH_SHOW_ALL, "pipeline"); + + /* Wait till pipeline encounters an error or EOS */ + g_print("Running...\n"); + g_main_loop_run(loop); + + /* Out of the main loop, clean up nicely */ + g_print("Returned, stopping playback\n"); + gst_element_set_state(pipeline, GST_STATE_NULL); + g_print("Deleting pipeline\n"); + gst_object_unref(GST_OBJECT(pipeline)); + g_source_remove(bus_watch_id); + g_main_loop_unref(loop); + if (_pose_file) { + fsetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, "]\n"); + + fclose(_pose_file); + } + + + return 0; +} diff --git a/deepstream-bodypose-3d/sources/nvdsinfer_custom_impl_BodyPose3DNet/Makefile b/deepstream-bodypose-3d/sources/nvdsinfer_custom_impl_BodyPose3DNet/Makefile new file mode 100755 index 0000000..ae9e133 --- /dev/null +++ b/deepstream-bodypose-3d/sources/nvdsinfer_custom_impl_BodyPose3DNet/Makefile @@ -0,0 +1,41 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +CC:= g++ + +CFLAGS:= -Wall -std=c++11 -shared -fPIC -Wno-error=deprecated-declarations +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes -I/usr/local/cuda-$(CUDA_VER)/include + + +LIBS:= -lnvinfer -lnvinfer_plugin +LFLAGS:= -Wl,--start-group $(LIBS) -Wl,--end-group + +SRCFILES:= nvdsinitinputlayers_BodyPose3DNet.cpp +TARGET_LIB:= libnvdsinfer_custom_impl_BodyPose3DNet.so + +all: $(TARGET_LIB) + +$(TARGET_LIB) : $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS) $(LFLAGS) + +clean: + rm -rf $(TARGET_LIB) diff --git a/deepstream-bodypose-3d/sources/nvdsinfer_custom_impl_BodyPose3DNet/nvdsinitinputlayers_BodyPose3DNet.cpp b/deepstream-bodypose-3d/sources/nvdsinfer_custom_impl_BodyPose3DNet/nvdsinitinputlayers_BodyPose3DNet.cpp new file mode 100755 index 0000000..8f89d81 --- /dev/null +++ b/deepstream-bodypose-3d/sources/nvdsinfer_custom_impl_BodyPose3DNet/nvdsinitinputlayers_BodyPose3DNet.cpp @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "nvdsinfer_custom_impl.h" +#include +/* Assumes only one input layer "im_info" needs to be initialized */ +bool NvDsInferInitializeInputLayers (std::vector const &inputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + unsigned int maxBatchSize) +{ + float scale_normalized_mean_limb_lengths[] = { + 0.5000, 0.5000, 1.0000, 0.8175, 0.9889, 0.2610, 0.7942, 0.5724, 0.5078, + 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3433, 0.8171, + 0.9912, 0.2610, 0.8259, 0.5724, 0.5078, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0000, 0.0000, 0.3422, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000}; + float mean_limb_lengths[] = { + 246.3427, 246.3427, 492.6854, 402.4380, 487.0321, 128.6856, 391.6295, + 281.9928, 249.9478, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0000, 169.1832, 402.2611, 488.1824, 128.6848, 407.5836, + 281.9897, 249.9489, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0000, 168.6137, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000}; + + //k_inv would change for camera parameters + float k_inv[] = {0.00124876620338, 0, -0.119881555525, + 0, 0.00124876620338, -0.159842074033, + 0, 0, 1}; + + float t_form_inv[] = {1.0, 0.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 0.0, 1.0}; + for (auto v : inputLayersInfo){ + if (!strcmp(v.layerName, "scale_normalized_mean_limb_lengths")){ + memcpy(v.buffer,scale_normalized_mean_limb_lengths,sizeof(float)*36); + } + if (!strcmp(v.layerName, "mean_limb_lengths")){ + memcpy(v.buffer,mean_limb_lengths,sizeof(float)*36); + } + if (!strcmp(v.layerName, "k_inv")){ + memcpy(v.buffer,k_inv,sizeof(float)*9); + } + if (!strcmp(v.layerName, "t_form_inv")){ + memcpy(v.buffer,t_form_inv,sizeof(float)*9); + } + } + + return true; +} + diff --git a/deepstream-bodypose-3d/streams/bodypose.mp4 b/deepstream-bodypose-3d/streams/bodypose.mp4 new file mode 100755 index 0000000..c3a533e --- /dev/null +++ b/deepstream-bodypose-3d/streams/bodypose.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46f38d405ef54ea9bee381bb794d32499e12911bd68f06d999372490ee64ca5a +size 3934589 diff --git a/deepstream-custom-tile-config/Makefile b/deepstream-custom-tile-config/Makefile new file mode 100644 index 0000000..7b16d6d --- /dev/null +++ b/deepstream-custom-tile-config/Makefile @@ -0,0 +1,29 @@ + +APP:= deepstream-custom-tile-config + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 gstreamer-video-1.0 glib-2.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) + +LIBS:= $(shell pkg-config --libs $(PKGS)) + +LIBS+= -lm -L/usr/local/cuda/lib64/ -lcudart +CFLAGS+= -I/usr/local/cuda/include + +all: $(APP) + +.o: .c $(INCS) Makefile + $(CC) -c $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +clean: + rm -rf $(OBJS) $(APP) + diff --git a/deepstream-custom-tile-config/README.md b/deepstream-custom-tile-config/README.md new file mode 100644 index 0000000..70ec926 --- /dev/null +++ b/deepstream-custom-tile-config/README.md @@ -0,0 +1,29 @@ +# DeepStream Custom Tiler Configuration Sample + +## Introduction +This sample demonstrates the usage of "custom-tile-config" of nvmultistreamtiler to customize the tiling positions and sizes of multiple videos within the display window. The rectangle display areas of every video can be configured by the "custom-tile-config" property of nvmultistreamtiler for CustomTileConfig struct. + +## Prerequisites +The sample works with DeepStream 9.0 GA or above version. Please follow [DeepStream SDK installation instruction](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_Installation.html) or use [DeepStream docker container](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_docker_containers.html) to prepare the DeepStream environment. + +## Running the Application +Download the source code and build + +`` +git clone https://github.com/NVIDIA-AI-IOT/deepstream_reference_apps.git +cd deepstream_reference_apps/deepstream-custom-tile-config +make +`` + +Run the sample with four video files and generate mp4 video for the output + +`` +./deepstream-custom-tile-config -i file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 --no-display +`` +Run the sample with four video files and dislay on the screen +`` +./deepstream-custom-tile-config -i file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 +`` + +## Known Issue +The video blending is not supported now. If there are overlapping parts of the videos, the overlapping parts will flicker. \ No newline at end of file diff --git a/deepstream-custom-tile-config/custom_tiler_cfg.c b/deepstream-custom-tile-config/custom_tiler_cfg.c new file mode 100644 index 0000000..a6e383b --- /dev/null +++ b/deepstream-custom-tile-config/custom_tiler_cfg.c @@ -0,0 +1,371 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "nvdstilerconfig.h" + +typedef struct { + GstElement *pipeline; + GstElement *streammux; + GstElement *tiler; + gboolean no_display; + guint num_sources; + /* Keep these alive for the lifetime of the pipeline */ + CustomTile *tiles_arr; + CustomTileConfig *cfg; +} AppCtx; + +static gboolean +bus_call(GstBus *bus, GstMessage *msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *)data; + switch (GST_MESSAGE_TYPE(msg)) { + case GST_MESSAGE_EOS: + g_printerr("End-of-stream\n"); + g_main_loop_quit(loop); + break; + case GST_MESSAGE_ERROR: { + GError *err = NULL; + gchar *debug = NULL; + gst_message_parse_error(msg, &err, &debug); + g_printerr("Error: %s\n", err ? err->message : "(unknown)"); + if (debug) g_printerr("Debug details: %s\n", debug); + g_clear_error(&err); + g_free(debug); + g_main_loop_quit(loop); + break; + } + default: + break; + } + return TRUE; +} + +static void +cb_newpad(GstElement *decodebin, GstPad *pad, gpointer user_data) +{ + GstCaps *caps = NULL; + const gchar *name = NULL; + GstStructure *str = NULL; + GstBin *source_bin = GST_BIN(user_data); + + caps = gst_pad_get_current_caps(pad); + if (!caps) caps = gst_pad_query_caps(pad, NULL); + if (!caps) return; + str = gst_caps_get_structure(caps, 0); + name = gst_structure_get_name(str); + + if (name && g_str_has_prefix(name, "video")) { + /* Only link NVMM pads */ + GstCapsFeatures *features = gst_caps_get_features(caps, 0); + if (features && gst_caps_features_contains(features, "memory:NVMM")) { + GstPad *ghost = gst_element_get_static_pad(GST_ELEMENT(source_bin), "src"); + if (ghost) { + if (!gst_ghost_pad_set_target(GST_GHOST_PAD(ghost), pad)) { + g_printerr("Failed to set ghost pad target\n"); + } + gst_object_unref(ghost); + } + } else { + g_printerr("Decodebin did not pick NVMM memory\n"); + } + } + if (caps) gst_caps_unref(caps); +} + +static void +decodebin_child_added(GstChildProxy *child_proxy, GObject *object, gchar *name, gpointer user_data) +{ + if (g_strrstr(name, "decodebin")) { + g_signal_connect(object, "child-added", G_CALLBACK(decodebin_child_added), user_data); + } +} + +static GstElement * +create_source_bin(guint index, const gchar *uri) +{ + gchar bin_name[32]; + g_snprintf(bin_name, sizeof(bin_name), "source-bin-%02u", index); + GstElement *bin = gst_bin_new(bin_name); + if (!bin) return NULL; + + GstElement *uri_decode_bin = gst_element_factory_make("uridecodebin", "uri-decode-bin"); + if (!uri_decode_bin) { + g_printerr("Unable to create uridecodebin\n"); + gst_object_unref(bin); + return NULL; + } + g_object_set(G_OBJECT(uri_decode_bin), "uri", uri, NULL); + g_signal_connect(uri_decode_bin, "pad-added", G_CALLBACK(cb_newpad), bin); + g_signal_connect(uri_decode_bin, "child-added", G_CALLBACK(decodebin_child_added), bin); + + gst_bin_add(GST_BIN(bin), uri_decode_bin); + + /* Create an initially no-target ghost pad that will be set in cb_newpad */ + GstPad *ghost = gst_ghost_pad_new_no_target("src", GST_PAD_SRC); + if (!ghost) { + g_printerr("Failed to add ghost pad\n"); + gst_object_unref(bin); + return NULL; + } + gst_element_add_pad(bin, ghost); + return bin; +} + +static void +build_custom_layout(AppCtx *app) +{ + /* Example asymmetric layout similar to the Python sample */ + guint n = app->num_sources; + if (n == 0) return; + + guint length = n; + CustomTile *tiles = g_new0(CustomTile, length); + + /* Default: 2x2 style base with one big-left if >=3 */ + if (length >= 1) { tiles[0].sourceId = 0; tiles[0].x = 0.00f; tiles[0].y = 0.00f; tiles[0].width = (n >= 3 ? 0.66f : 1.0f); tiles[0].height = 1.00f; } + if (length >= 2) { tiles[1].sourceId = 1; tiles[1].x = (n >= 3 ? 0.66f : 0.00f); tiles[1].y = 0.00f; tiles[1].width = (n >= 3 ? 0.34f : 1.0f - tiles[0].x); tiles[1].height = (n >= 3 ? 0.50f : 1.0f); } + if (length >= 3) { tiles[2].sourceId = 2; tiles[2].x = 0.66f; tiles[2].y = 0.50f; tiles[2].width = 0.34f; tiles[2].height = 0.50f; } + if (length >= 4) { tiles[3].sourceId = 3; tiles[3].x = 0.33f; tiles[3].y = 0.33f; tiles[3].width = 0.33f; tiles[3].height = 0.33f; } + + app->tiles_arr = tiles; + app->cfg = g_new0(CustomTileConfig, 1); + app->cfg->tiles = app->tiles_arr; + app->cfg->length = length; +} + +static gboolean +update_layout_cb(gpointer user_data) +{ + AppCtx *app = (AppCtx *)user_data; + if (!app || !app->cfg || !app->cfg->tiles || app->cfg->length == 0) return G_SOURCE_REMOVE; + + /* Make the first tile full screen as a demo update */ + app->cfg->tiles[0].x = 0.0f; + app->cfg->tiles[0].y = 0.0f; + app->cfg->tiles[0].width = 1.0f; + app->cfg->tiles[0].height = 1.0f; + + g_object_set(G_OBJECT(app->tiler), "custom-tile-config", (gpointer)app->cfg, NULL); + g_printerr("Custom layout updated at runtime\n"); + return G_SOURCE_REMOVE; /* one-shot */ +} + +int +main(int argc, char *argv[]) +{ + GMainLoop *loop = NULL; + AppCtx app; + int current_device = -1; + gboolean enc_hw_support = TRUE; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + if (prop.integrated) { + FILE* ptr; + char device_name[50]; + ptr = fopen("/proc/device-tree/model", "r"); + + if(ptr){ + while (fgets(device_name, 50, ptr) != NULL) { + if (strstr(device_name,"Orin") && (strstr(device_name,"Nano"))) + enc_hw_support = FALSE; + } + fclose(ptr); + } + } + + memset(&app, 0, sizeof(app)); + + gst_init(&argc, &argv); + + /* Simple arg handling: program -i uri1 uri2 ... [--no-display] */ + GPtrArray *uris = g_ptr_array_new_with_free_func(g_free); + for (int i = 1; i < argc; ++i) { + if (g_strcmp0(argv[i], "-i") == 0 || g_strcmp0(argv[i], "--input") == 0) { + for (int j = i + 1; j < argc && argv[j][0] != '-'; ++j) { + g_ptr_array_add(uris, g_strdup(argv[j])); + i = j; + } + } else if (g_strcmp0(argv[i], "--no-display") == 0) { + app.no_display = TRUE; + } + } + + if (uris->len == 0) { + g_printerr("Usage: %s -i [uri2 ...] [--no-display]\n", argv[0]); + g_ptr_array_unref(uris); + return -1; + } + + app.num_sources = uris->len; + + app.pipeline = gst_pipeline_new("ds-custom-tiler"); + if (!app.pipeline) { + g_printerr("Failed to create pipeline\n"); + g_ptr_array_unref(uris); + return -1; + } + + app.streammux = gst_element_factory_make("nvstreammux", "stream-muxer"); + if (!app.streammux) { + g_printerr("Failed to create nvstreammux\n"); + gst_object_unref(app.pipeline); + g_ptr_array_unref(uris); + return -1; + } + g_object_set(G_OBJECT(app.streammux), + "width", 1920, + "height", 1080, + "batch-size", app.num_sources, + "batched-push-timeout", 33000, + NULL); + + gst_bin_add(GST_BIN(app.pipeline), app.streammux); + + for (guint i = 0; i < app.num_sources; ++i) { + GstElement *src_bin = create_source_bin(i, (const gchar *)g_ptr_array_index(uris, i)); + if (!src_bin) { + g_printerr("Failed to create source bin %u\n", i); + gst_object_unref(app.pipeline); + g_ptr_array_unref(uris); + return -1; + } + gst_bin_add(GST_BIN(app.pipeline), src_bin); + + gchar pad_name[32]; + g_snprintf(pad_name, sizeof(pad_name), "sink_%u", i); + GstPad *sinkpad = gst_element_request_pad_simple(app.streammux, pad_name); + GstPad *srcpad = gst_element_get_static_pad(src_bin, "src"); + if (!sinkpad || !srcpad) { + g_printerr("Failed to get pads for source %u\n", i); + if (sinkpad) gst_object_unref(sinkpad); + if (srcpad) gst_object_unref(srcpad); + gst_object_unref(app.pipeline); + g_ptr_array_unref(uris); + return -1; + } + if (gst_pad_link(srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr("Failed to link source bin to streammux for %u\n", i); + gst_object_unref(sinkpad); + gst_object_unref(srcpad); + gst_object_unref(app.pipeline); + g_ptr_array_unref(uris); + return -1; + } + gst_object_unref(sinkpad); + gst_object_unref(srcpad); + } + + GstElement *queue1 = gst_element_factory_make("queue", "queue1"); + GstElement *queue2 = gst_element_factory_make("queue", "queue2"); + GstElement *queue3 = gst_element_factory_make("queue", "queue3"); + GstElement *queue4 = gst_element_factory_make("queue", "queue4"); + GstElement *queue5 = gst_element_factory_make("queue", "queue5"); + + GstElement *pgie = gst_element_factory_make("queue", "primary-inference"); /* placeholder */ + app.tiler = gst_element_factory_make("nvmultistreamtiler", "nvtiler"); + GstElement *nvvidconv = gst_element_factory_make("nvvideoconvert", "convertor"); + GstElement *nvosd = gst_element_factory_make("nvdsosd", "onscreendisplay"); + GstElement *sink = NULL; + if(app.no_display) { + sink = gst_element_factory_make("nvvideoencfilesinkbin", "fakesink"); + } else { + if (prop.integrated) { + sink = gst_element_factory_make("nv3dsink", "nvvideo-renderer"); + } else { +#ifdef __aarch64__ + sink = gst_element_factory_make("nv3dsink", "nvvideo-renderer"); +#else + sink = gst_element_factory_make("nveglglessink", "nvvideo-renderer"); +#endif + } + } + + if (!queue1 || !queue2 || !queue3 || !queue4 || !queue5 || !pgie || !app.tiler || !nvvidconv || !nvosd || !sink) { + g_printerr("Failed to create one or more elements\n"); + gst_object_unref(app.pipeline); + g_ptr_array_unref(uris); + return -1; + } + + g_object_set(G_OBJECT(nvosd), "process-mode", 0, "display-text", 1, NULL); + g_object_set(G_OBJECT(app.tiler), "square-seq-grid", FALSE, NULL); + g_object_set(G_OBJECT(app.tiler), "compute-hw", 1, NULL); + if (!app.no_display) g_object_set(G_OBJECT(sink), "qos", FALSE, NULL); + if (app.no_display) { + g_object_set(G_OBJECT(sink), "output-file", "/tmp/tile_out.mp4", "enc-type", 1, NULL); + if (!enc_hw_support){ + g_object_set(G_OBJECT(sink), "enc-type", 1, NULL); + } + } + + /* Initial custom asymmetric layout */ + build_custom_layout(&app); + if (app.cfg) { + g_object_set(G_OBJECT(app.tiler), "custom-tile-config", (gpointer)app.cfg, NULL); + //g_timeout_add_seconds(5, update_layout_cb, &app); + } + + gst_bin_add_many(GST_BIN(app.pipeline), + queue1, pgie, queue2, app.tiler, queue3, nvvidconv, queue4, nvosd, queue5, sink, NULL); + + if (!gst_element_link_many(app.streammux, queue1, pgie, queue2, app.tiler, queue3, nvvidconv, queue4, nvosd, queue5, sink, NULL)) { + g_printerr("Failed to link elements\n"); + gst_object_unref(app.pipeline); + g_ptr_array_unref(uris); + return -1; + } + + /* Rows/Columns computed like the Python sample */ + guint rows = (guint)floor(sqrt((double)app.num_sources)); + if (rows == 0) rows = 1; + guint cols = (guint)ceil(((double)app.num_sources) / (double)rows); + g_object_set(G_OBJECT(app.tiler), "rows", rows, "columns", cols, "width", 1280, "height", 720, NULL); + + loop = g_main_loop_new(NULL, FALSE); + GstBus *bus = gst_element_get_bus(app.pipeline); + gst_bus_add_signal_watch(bus); + g_signal_connect(G_OBJECT(bus), "message", G_CALLBACK(bus_call), loop); + gst_object_unref(bus); + + g_print("Now playing...\n"); + for (guint i = 0; i < uris->len; ++i) { + g_print("%u: %s\n", i, (gchar *)g_ptr_array_index(uris, i)); + } + + gst_element_set_state(app.pipeline, GST_STATE_PLAYING); + g_main_loop_run(loop); + + gst_element_set_state(app.pipeline, GST_STATE_NULL); + g_main_loop_unref(loop); + + /* Cleanup */ + if (app.tiles_arr) g_free(app.tiles_arr); + if (app.cfg) g_free(app.cfg); + gst_object_unref(app.pipeline); + g_ptr_array_unref(uris); + return 0; +} diff --git a/deepstream-custom-tile-config/nvdstilerconfig.h b/deepstream-custom-tile-config/nvdstilerconfig.h new file mode 100644 index 0000000..b9e21f5 --- /dev/null +++ b/deepstream-custom-tile-config/nvdstilerconfig.h @@ -0,0 +1,93 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef _NVDSTILER_CONFIG_H_ +#define _NVDSTILER_CONFIG_H_ + +/** + * @defgroup ds_nvtiler_custom_api NvTiler Custom canvas API module + * NvTiler CustomTile configuration structures + */ + +#include + +/** + * Holds information about an individual tile in the custom canvas. + * One tile is the space on canvas where nvmultistreamtiler + * render an individual source + */ +typedef struct +{ + /** sourceId identifying the video source in a DeepStream pipeline */ + uint32_t sourceId; + /** offset from left in the unit - 1/100 or + * percentage of canvas-width */ + float x; + /** offset from top in the unit - 1/100 or + * percentage of canvas-height */ + float y; + /** tile width in the unit - 1/100 or + * percentage of canvas-width */ + float width; + /** tile height in the unit - 1/100 or + * percentage of canvas-height */ + float height; +}CustomTile; + +/** + * Holds information about the custom tile canvas + * A pointer to this memory (transfer-none) shall be + * set on custom-tile-config property on nvmultistreamtiler plugin. + * Please check `gst-inspect-1.0 nvmultistreamtiler` for more info. + * Note 1: This data structure shall be filled with + * individual tile resolution for all involved sources. + * Note 2: custom-tile-config property can be configured dynamically + * while the pipeline is running. + * Note 3: To remove sources from the canvas (example: EOS), user shall set the + * custom-tile-config property again by removing entry in + * the array CustomTileConfig->tiles and update CustomTileConfig->length + */ +typedef struct +{ + /** custom tile-level config array + * NOTE: nullable + * Used to customize the tile resolution per source + * If used, user shall pass configuration for all the + * individual tiles in [rows X columns] canvas + */ + CustomTile* tiles; + uint32_t length; /**< length of tiles config array */ +}CustomTileConfig; + +/** + * Holds the Tiler Canvas Configuration + * Note: The user cannot directly set this on nvmultistreamtiler + * User shall leverage properties on nvmultistreamtiler plugin + * to configure the canvas. + * Please check `gst-inspect-1.0 nvmultistreamtiler` for more info. + */ +typedef struct +{ + uint32_t width; /**< canvas width */ + uint32_t height; /**< canvas height */ + uint32_t columns; /**< #columns of tiles in canvas */ + uint32_t rows; /**< #rows of tiles in canvas */ + uint32_t gpuId; /**< #gpuId to be used for stream creation */ + CustomTileConfig customTileConfig; +}TilerConfig; + + +#endif /**< _NVDSTILER_CONFIG_H_ */ diff --git a/deepstream-dynamicsrcbin-test/Makefile b/deepstream-dynamicsrcbin-test/Makefile new file mode 100644 index 0000000..64e91a1 --- /dev/null +++ b/deepstream-dynamicsrcbin-test/Makefile @@ -0,0 +1,62 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +APP:= deepstream-dynamicsrcbin-test + +TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -) + +NVDS_VERSION:=9.0 + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream-$(NVDS_VERSION)/lib/ +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream-$(NVDS_VERSION)/bin/ + +ifeq ($(TARGET_DEVICE),aarch64) + CFLAGS:= -DPLATFORM_TEGRA +endif + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= -I../../../includes \ + -I /usr/local/cuda-$(CUDA_VER)/include + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) + +LIBS:= $(shell pkg-config --libs $(PKGS)) + + + +all: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) + + diff --git a/deepstream-dynamicsrcbin-test/README b/deepstream-dynamicsrcbin-test/README new file mode 100644 index 0000000..991924b --- /dev/null +++ b/deepstream-dynamicsrcbin-test/README @@ -0,0 +1,177 @@ +***************************************************************************** +* SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +* SPDX-License-Identifier: LicenseRef-NvidiaProprietary +* +* NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +* property and proprietary rights in and to this material, related +* documentation and any modifications thereto. Any use, reproduction, +* disclosure or distribution of this material and related documentation +* without an express license agreement from NVIDIA CORPORATION or +* its affiliates is strictly prohibited. +***************************************************************************** + +***************************************************************************** + deepstream-dynamicsrcbin-test-app + README +***************************************************************************** + +=============================================================================== +1. Prerequisites: +=============================================================================== +Please follow instructions in the apps/sample_apps/deepstream-app/README on how +to install the prerequisites for Deepstream SDK, the DeepStream SDK itself and the +apps. + +You must have the following development packages installed: + GStreamer-1.0 + GStreamer-1.0 Base Plugins + GStreamer-1.0 gstrtspserver + X11 client-side library + +To install these packages, execute the following command: + sudo apt-get install libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev \ + libgstrtspserver-1.0-dev libx11-dev + +=============================================================================== +2. Purpose: +=============================================================================== + +This application demonstrates the capabilities of the **nvdsdynamicsrcbin** element, +specifically designed for high decoder throughput scenarios. The key innovation +of this element is that it creates a single pipeline that keeps the decoder +active at all times, avoiding decoder initialization overhead for every new stream. + +**Key Benefits:** +- **Eliminates Decoder Initialization Overhead**: When you add N streams to a + traditional pipeline, each stream typically requires its own decoder instance + with initialization time. The nvdsdynamicsrcbin element maintains a single + active decoder, significantly reducing latency when switching between streams. + +- **Improved Throughput**: By keeping the decoder warm and ready, the element + can handle rapid stream switching without the performance penalty of repeated + decoder initialization. + +- **Dynamic Source Management**: The element supports real-time addition and + removal of video sources without pipeline reconstruction. + +=============================================================================== +3. Element Architecture: +=============================================================================== + +The **nvdsdynamicsrcbin** element is a custom GStreamer bin that contains: +- **filesrc**: Source element for reading video files +- **queue_filesrc**: Buffering queue for source data +- **parsebin**: Media format detection and parsing +- **queue_parsebin**: Buffering queue for parsed data +- **nvv4l2decoder**: Hardware decoder (kept active throughout) + +The element maintains internal queues and maps to track active sources and their +corresponding file paths, enabling seamless switching between multiple video streams. + +=============================================================================== +4. Supported Signals: +=============================================================================== + +The nvdsdynamicsrcbin element supports three key signals for dynamic source management: + +### 4.1 add-source Signal +**Purpose**: Adds a new video source to the dynamic source bin +**Parameters**: +- `file_path` (string): Path to the video file +- `source_id` (integer): Unique identifier for the source + +**Usage**: +```c +g_signal_emit_by_name(dynamicsrcbin, "add-source", "/path/to/video.mp4", source_id); +``` + +**Behavior**: +- Validates the file path exists +- Prevents duplicate source IDs +- Stores source_id -> file_path mapping +- Adds source to internal tracking queues +- For the first source, links elements to the pipeline +- Subsequent sources are queued for processing + +### 4.2 remove-source Signal +**Purpose**: Removes a video source from the dynamic source bin +**Parameters**: +- `source_id` (integer): ID of the source to remove + +**Usage**: +```c +g_signal_emit_by_name(dynamicsrcbin, "remove-source", source_id); +``` + +**Behavior**: +- If the source is currently active (head of queue), sends EOS event +- If the source is queued but not active, removes it from tracking +- Cleans up internal data structures +- Posts file change message to notify downstream elements + +### 4.3 terminate Signal +**Purpose**: Terminates the entire pipeline gracefully +**Parameters**: None + +**Usage**: +```c +g_signal_emit_by_name(dynamicsrcbin, "terminate"); +``` + +**Behavior**: +- Sends EOS event from the decoder element +- Posts termination message to exit the pipeline +- Cleans up all resources and stops processing + +=============================================================================== +5. Custom Events and Metadata: +=============================================================================== + +The element generates custom events to track source changes: + +### 5.1 Custom Stream Start Event +- **Event Name**: "custom-stream-start-event" +- **Contains**: source-id parameter +- **Purpose**: Notifies downstream elements when a new source begins processing + +### 5.2 Custom EOS Event +- **Event Name**: "custom-eos-event" +- **Contains**: source-id parameter +- **Purpose**: Notifies when a source completes processing + +### 5.3 Source Metadata +The element attaches custom metadata to each buffer containing: +- **chunk_id**: The source ID that generated the buffer +- **frame_id**: Sequential frame counter for the current source + +=============================================================================== +6. To compile: +=============================================================================== + + $ Set CUDA_VER in the MakeFile as per platform. + For x86, CUDA_VER=13.1 + For jetson, CUDA_VER=13.0 + $ sudo make (sudo not required in case of docker containers) + +=============================================================================== +7. Usage: +=============================================================================== + +The application demonstrates dynamic source management with the following workflow: + +1. **Pipeline Setup**: Creates a pipeline with nvdsdynamicsrcbin and fakesink +2. **Source Addition**: Adds multiple video sources with unique IDs +3. **Dynamic Processing**: Processes sources sequentially with active decoder +4. **Monitoring**: Tracks frame processing and source changes +5. **Cleanup**: Terminates pipeline gracefully + +**Running the Application**: +```bash +$ ./deepstream-dynamicsrcbin-test-app +``` + +**Configuration**: +- Modify `N_CHUNKS` in the source code to change the number of sources +- Update `VIDEO_FILE` path to use different video files +- Adjust timing parameters in the signal thread for different test scenarios + diff --git a/deepstream-dynamicsrcbin-test/deepstream_dynamicsrcbin_test.c b/deepstream-dynamicsrcbin-test/deepstream_dynamicsrcbin_test.c new file mode 100644 index 0000000..d427a27 --- /dev/null +++ b/deepstream-dynamicsrcbin-test/deepstream_dynamicsrcbin_test.c @@ -0,0 +1,433 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include +#include + +/* Configuration constants */ +#define N_CHUNKS 2 +#define VIDEO_FILE "/opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4" + +/** + * @brief Application data structure to hold pipeline components and state + */ +typedef struct { + GstElement *pipeline; /* Main GStreamer pipeline */ + GstElement *src; /* nvdsdynamicsrcbin source element */ + GstElement *sink; /* fakesink element for testing */ + GMainLoop *loop; /* Main event loop */ + pthread_t signal_thread; /* Thread for sending dynamic source signals */ + gboolean thread_running; /* Flag to track thread state */ +} AppData; + +/* Global variables for tracking frame processing */ +guint frame_count = 0; +gint current_source_id = -1; + +/** + * @brief Pad probe callback for sink input to monitor frame processing + * + * This callback is attached to the sink pad to: + * - Count processed frames + * - Handle custom stream start events from nvdsdynamicsrcbin + * - Track source ID changes + * + * @param pad The sink pad being probed + * @param info Probe information containing buffer or event data + * @param user_data User data (unused in this implementation) + * @return GST_PAD_PROBE_OK to continue processing + */ +static GstPadProbeReturn sink_input_buffer_callback(GstPad *pad, GstPadProbeInfo *info, gpointer user_data) { + if (GST_PAD_PROBE_INFO_TYPE(info) & GST_PAD_PROBE_TYPE_BUFFER) { + frame_count++; + g_print("Frame Number: %d, Current Source ID: %d\n", frame_count, current_source_id); + } else if (GST_PAD_PROBE_INFO_TYPE(info) & GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM) { + GstEvent *event = GST_PAD_PROBE_INFO_EVENT(info); + /* Check for custom stream start event from nvdsdynamicsrcbin */ + if (GST_EVENT_TYPE(event) == GST_EVENT_CUSTOM_DOWNSTREAM) { + const GstStructure *structure = gst_event_get_structure(event); + if (structure && gst_structure_has_name(structure, "custom-stream-start-event")) { + gint source_id; + if (gst_structure_get_int(structure, "source-id", &source_id)) { + g_print("Source change event received for source ID: %d\n", source_id); + current_source_id = source_id; + /* Reset frame count for new source */ + frame_count = 0; + } + } + } + } + return GST_PAD_PROBE_OK; +} + +/** + * @brief Utility function to find an element by factory name within a bin + * + * @param bin The GStreamer bin to search in + * @param factory_name The factory name of the element to find + * @return Pointer to the found element (with increased ref count) or NULL if not found + */ +GstElement* get_element_by_factory_name(GstBin *bin, const gchar *factory_name) { + GstElement *element = NULL; + GstIterator *iterator = gst_bin_iterate_all_by_element_factory_name(GST_BIN(bin), factory_name); + GValue item = G_VALUE_INIT; + + /* Get first matching element */ + if (gst_iterator_next(iterator, &item) == GST_ITERATOR_OK) { + element = GST_ELEMENT(g_value_get_object(&item)); + gst_object_ref(element); /* Increase ref count since we'll be using it */ + g_value_reset(&item); + } + + gst_iterator_free(iterator); + return element; +} + +/** + * @brief Load a new file into the dynamic source bin + * + * Creates and adds filesrc, queue, and parsebin elements to the nvdsdynamicsrcbin + * and sets them to PLAYING state. + * + * @param dynamicsrcbin The nvdsdynamicsrcbin element + * @param current_file Path to the file to load + * @return TRUE if successful, FALSE otherwise + */ +static gboolean load_file(GstElement *dynamicsrcbin, gchar *current_file) { + /* Create new elements for the file */ + GstElement *filesrc = gst_element_factory_make("filesrc", "source"); + GstElement *queue = gst_element_factory_make("queue", "queue"); + GstElement *parsebin = gst_element_factory_make("parsebin", "parsebin"); + + if (!filesrc || !queue || !parsebin) { + g_print("One of the elements not successfully created\n"); + return FALSE; + } + + /* Update filesrc location */ + g_object_set(filesrc, "location", current_file, NULL); + g_print("Playing next file: %s\n", current_file); + + /* Add the elements to dynamicsrcbin */ + gst_bin_add_many(GST_BIN(dynamicsrcbin), filesrc, queue, parsebin, NULL); + + /* Link elements */ + gst_element_link_many(filesrc, queue, parsebin, NULL); + + /* Set elements to PLAYING state */ + gst_element_set_state(parsebin, GST_STATE_PLAYING); + gst_element_set_state(queue, GST_STATE_PLAYING); + gst_element_set_state(filesrc, GST_STATE_PLAYING); + + return TRUE; +} + +/** + * @brief Timeout callback to attempt loading a file + * + * This function is called periodically to check if a file is ready to be loaded + * into the nvdsdynamicsrcbin. + * + * @param data Pointer to the nvdsdynamicsrcbin element + * @return TRUE to continue calling, FALSE to stop + */ +static gboolean try_load_file(gpointer data) { + GstElement *dynamicsrcbin = GST_ELEMENT(data); + int current_id = -1; + gchar *current_file = NULL; + + g_object_get(G_OBJECT(dynamicsrcbin), "current-id", ¤t_id, NULL); + g_object_get(G_OBJECT(dynamicsrcbin), "current-file", ¤t_file, NULL); + + if (!current_file || current_id != -1) { + g_print("Waiting for file to come...\n"); + return TRUE; + } + + if (!load_file(dynamicsrcbin, current_file)) { + g_print("Failed to load file: %s\n", current_file); + } + + g_free(current_file); + /* File loaded, no need to try again */ + return FALSE; +} + +/** + * @brief Handle dynamic source message from nvdsdynamicsrcbin + * + * This function is called when the nvdsdynamicsrcbin sends a file change message. + * It removes the old elements and loads the new file. + * + * @param dynamicsrcbin The nvdsdynamicsrcbin element + * @return TRUE if successful, FALSE otherwise + */ +static gboolean handle_dynamic_source_message(GstElement *dynamicsrcbin) { + int current_id = -1; + gchar *current_file = NULL; + GstElement *filesrc = get_element_by_factory_name(GST_BIN(dynamicsrcbin), "filesrc"); + GstElement *queue = get_element_by_factory_name(GST_BIN(dynamicsrcbin), "queue"); + GstElement *parsebin = get_element_by_factory_name(GST_BIN(dynamicsrcbin), "parsebin"); + + if (!filesrc || !queue || !parsebin) { + g_print("One of the elements not found\n"); + } else { + /* Stop and remove old elements */ + gst_element_set_state(parsebin, GST_STATE_NULL); + gst_element_set_state(queue, GST_STATE_NULL); + gst_element_set_state(filesrc, GST_STATE_NULL); + + gst_bin_remove(GST_BIN(dynamicsrcbin), filesrc); + gst_bin_remove(GST_BIN(dynamicsrcbin), queue); + gst_bin_remove(GST_BIN(dynamicsrcbin), parsebin); + + /* Get current file and ID from nvdsdynamicsrcbin */ + g_object_get(G_OBJECT(dynamicsrcbin), "current-id", ¤t_id, NULL); + g_object_get(G_OBJECT(dynamicsrcbin), "current-file", ¤t_file, NULL); + g_print("Current ID: %d, Current File: %s\n", current_id, current_file); + + if (current_file && current_id != -1) { + gboolean ret = load_file(dynamicsrcbin, current_file); + g_free(current_file); + return ret; + } + + /* Schedule retry if file not ready */ + g_timeout_add(1000, try_load_file, dynamicsrcbin); + } + return TRUE; +} + +/** + * @brief Bus callback to handle GStreamer messages + * + * Handles various GStreamer bus messages including: + * - EOS (End of Stream) + * - Warnings and Errors + * - Application messages for dynamic source changes + * - Latency messages + * + * @param bus The GStreamer bus + * @param msg The message to handle + * @param data Pointer to AppData structure + * @return TRUE to continue receiving messages + */ +static gboolean bus_call(GstBus *bus, GstMessage *msg, gpointer data) { + AppData *app = (AppData *)data; + + switch (GST_MESSAGE_TYPE(msg)) { + case GST_MESSAGE_EOS: { + g_print("End of stream\n"); + g_main_loop_quit(app->loop); + break; + } + case GST_MESSAGE_WARNING: { + gchar *debug = NULL; + GError *error = NULL; + gst_message_parse_warning(msg, &error, &debug); + g_printerr("WARNING from element %s: %s\n", + GST_OBJECT_NAME(msg->src), error->message); + g_free(debug); + g_printerr("Warning: %s\n", error->message); + g_error_free(error); + break; + } + case GST_MESSAGE_ERROR: { + gchar *debug = NULL; + GError *error = NULL; + gst_message_parse_error(msg, &error, &debug); + g_printerr("ERROR from element %s: %s\n", + GST_OBJECT_NAME(msg->src), error->message); + if (debug) + g_printerr("Error details: %s\n", debug); + g_free(debug); + g_error_free(error); + g_main_loop_quit(app->loop); + break; + } + case GST_MESSAGE_APPLICATION: { + /* Handle dynamic source change messages from nvdsdynamicsrcbin */ + const GstStructure *str = gst_message_get_structure(msg); + if (gst_structure_has_name(str, "dynamic-src-bin-file-change")) { + GstElement *dynamicsrcbin = get_element_by_factory_name(GST_BIN(app->pipeline), "nvdsdynamicsrcbin"); + if (!dynamicsrcbin) { + g_printerr("Failed to get dynamicsrcbin\n"); + return FALSE; + } + handle_dynamic_source_message(dynamicsrcbin); + gst_object_unref(dynamicsrcbin); + } + break; + } + case GST_MESSAGE_LATENCY: { + /* Recalculate latency when receiving latency message */ + gst_bin_recalculate_latency(GST_BIN(app->pipeline)); + break; + } + default: + break; + } + return TRUE; +} + +/** + * @brief Test function to add and immediately remove a source + * + * Demonstrates how to add a source and then remove it before completion. + * + * @param data Pointer to AppData structure + * @param source_id The source ID to add and remove + */ +void add_remove_source_test(void *data, gint source_id) { + AppData *app = (AppData *)data; + g_signal_emit_by_name(app->src, "add-source", VIDEO_FILE, source_id); + g_print("Thread: Emitted signal for ID: %u, File: %s\n", source_id, VIDEO_FILE); + g_usleep(500000); /* 0.5 second delay */ + g_signal_emit_by_name(app->src, "remove-source", source_id); + g_print("Thread: Emitted signal for ID: %u, File: %s\n", source_id, VIDEO_FILE); + g_usleep(500000); /* 0.5 second delay */ +} + +/** + * @brief Test function to add a source + * + * @param data Pointer to AppData structure + * @param source_id The source ID to add + */ +void add_source_test(void *data, gint source_id) { + AppData *app = (AppData *)data; + g_signal_emit_by_name(app->src, "add-source", VIDEO_FILE, source_id); + g_print("Thread: Emitted signal for ID: %u, File: %s\n", source_id, VIDEO_FILE); +} + +/** + * @brief Thread function to send dynamic source signals + * + * This thread demonstrates how to dynamically add sources to the nvdsdynamicsrcbin + * element. It adds sources sequentially and then terminates the pipeline. + * + * @param data Pointer to AppData structure + * @return NULL + */ +static void* signal_thread_func(void *data) { + AppData *app = (AppData *)data; + + /* Wait for the pipeline to be ready */ + g_usleep(1000000); /* 1 second delay */ + + /* Add sources sequentially */ + for (gint source_id = 0; source_id < N_CHUNKS; source_id++) { + add_source_test(app, source_id); + /* Uncomment the line below to see how to remove source before it completes */ + /* add_remove_source_test(app, source_id); */ + } + + g_usleep(10000000); /* 10 second delay */ + /* Terminate the pipeline - for demonstration purposes only */ + /* Use when you don't need to add more sources and don't want to wait for completion */ + g_signal_emit_by_name(app->src, "terminate"); + + g_print("Signal thread completed\n"); + app->thread_running = FALSE; + return NULL; +} + +/** + * @brief Main function + * + * Sets up the GStreamer pipeline with nvdsdynamicsrcbin and fakesink, + * creates a signal thread for dynamic source management, and runs the main loop. + * + * @param argc Number of command line arguments + * @param argv Array of command line arguments + * @return 0 on success, -1 on failure + */ +int main(int argc, char *argv[]) { + AppData app; + guint bus_watch_id; + GstBus *bus; + + /* Initialize GStreamer */ + gst_init(NULL, NULL); + app.loop = g_main_loop_new(NULL, FALSE); + + /* Create pipeline elements */ + app.pipeline = gst_pipeline_new("dynamicsrcbin-test"); + app.src = gst_element_factory_make("nvdsdynamicsrcbin", "src"); + app.sink = gst_element_factory_make("fakesink", "sink"); + + if (!app.src || !app.sink) { + g_printerr("Failed to create elements\n"); + return -1; + } + + /* Configure sink element properties */ + g_object_set(app.sink, "sync", 0, NULL); + g_object_set(app.sink, "qos", 0, NULL); + + /* Set up bus monitoring */ + bus = gst_pipeline_get_bus(GST_PIPELINE(app.pipeline)); + bus_watch_id = gst_bus_add_watch(bus, bus_call, &app); + + /* Add elements to pipeline and link them */ + gst_bin_add_many(GST_BIN(app.pipeline), app.src, app.sink, NULL); + + if (!gst_element_link_many(app.src, app.sink, NULL)) { + g_printerr("Failed to link nvdsdynamicsrcbin to fakesink\n"); + return -1; + } + + /* Add probe to monitor frame processing */ + GstPad *sink_pad = gst_element_get_static_pad(app.sink, "sink"); + gst_pad_add_probe(sink_pad, + GST_PAD_PROBE_TYPE_BUFFER | GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, + (GstPadProbeCallback)sink_input_buffer_callback, + NULL, + NULL); + gst_object_unref(sink_pad); + + /* Set pipeline to PLAYING state */ + gst_element_set_state(app.pipeline, GST_STATE_PLAYING); + + /* Create and start the signal thread */ + app.thread_running = TRUE; + if (pthread_create(&app.signal_thread, NULL, signal_thread_func, &app) != 0) { + g_printerr("Failed to create signal thread\n"); + return -1; + } + g_print("Signal thread created and started\n"); + + /* Add initial sources to the nvdsdynamicsrcbin */ + for (gint source_id = 0; source_id < N_CHUNKS; source_id++) { + g_signal_emit_by_name(app.src, "add-source", VIDEO_FILE, source_id); + g_print("Current ID: %u\n", source_id); + } + + g_print("Running main loop...\n"); + g_main_loop_run(app.loop); + + /* Wait for signal thread to complete if it's still running */ + if (app.thread_running) { + g_print("Waiting for signal thread to complete...\n"); + pthread_join(app.signal_thread, NULL); + } + + /* Cleanup */ + g_print("Stopping playback\n"); + gst_element_set_state(app.pipeline, GST_STATE_NULL); + gst_object_unref(GST_OBJECT(app.pipeline)); + gst_object_unref(bus); + g_source_remove(bus_watch_id); + g_main_loop_unref(app.loop); + + return 0; +} \ No newline at end of file diff --git a/deepstream-ipc-test-sr/Makefile b/deepstream-ipc-test-sr/Makefile new file mode 100644 index 0000000..c94c440 --- /dev/null +++ b/deepstream-ipc-test-sr/Makefile @@ -0,0 +1,79 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +APP:= deepstream-ipc-test-app + +TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -) +ifeq ($(TARGET_DEVICE),aarch64) + CFLAGS+= -DPLATFORM_TEGRA +endif + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/bin/ + +SRCS:= $(wildcard *.c) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes \ + -I /usr/local/cuda-$(CUDA_VER)/include + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) + +LIBS:= $(shell pkg-config --libs $(PKGS)) + +LIBS+= -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart -lnvdsgst_helper -lm \ + -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvds_yml_parser -lgstapp-1.0 \ + -lcuda -Wl,-rpath,$(LIB_INSTALL_DIR) + +all: $(APP) + +%.o: %.c $(INCS) + $(CC) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) + $(CC) -o $(APP) $(OBJS) $(LIBS) + make -C video_template_impl + make -C latency_serialization + @echo "Build complete." + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + make -C video_template_impl install + make -C latency_serialization install + @echo "Install complete." + +clean: + make -C video_template_impl clean + make -C latency_serialization clean + rm -rf $(OBJS) $(APP) + @echo "Clean complete." + +.PHONY: all diff --git a/deepstream-ipc-test-sr/README.md b/deepstream-ipc-test-sr/README.md new file mode 100644 index 0000000..0abc7d3 --- /dev/null +++ b/deepstream-ipc-test-sr/README.md @@ -0,0 +1,164 @@ +# IPC Test Supper Resolution +## Introduction +This sample demonstrates how to Zero-copy share decoded buffers over IPC and how to integarte Super-Resolution +model. This sample can support Jetson and DGPU platform. +The client pipeline looks like "......-> nvstreammux -> pgie ->nvvideoconvert -> capsfilter -> nvvideotemplate + ......". + +## Prerequisites + +Please follow instructions in the /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-app/README on how +to install the prerequisites for the Deepstream SDK, the DeepStream SDK itself, and the apps. + +You must have the following development packages installed + GStreamer-1.0 + GStreamer-1.0 Base Plugins + GStreamer-1.0 gstrtspserver + X11 client-side library + +To install these packages, execute the following command: + sudo apt-get install libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev \ + libgstrtspserver-1.0-dev libx11-dev + +## Build + +```bash + $ Set CUDA_VER in the MakeFile as per platform. + For x86, CUDA_VER=13.1 + For Jetson, CUDA_VER=13.0 + $ sudo make +``` + +NOTE: To compile the sources, run make with "sudo" or root permission. + To improve performance on specific GPUs, please add "-gencode=arch=compute_xx,code=sm_xx" in Makefile. Computing capability can be found in this link https://developer.nvidia.com/zh-cn/cuda-gpus#compute. + +## Generate super resolution model + +The model is from pytorch [code](https://github.com/pytorch/tutorials/blob/5efa2e52aafdd94ef9ae6fbfa8c63fe888a15374/advanced_source/super_resolution_with_onnxruntime.py). +Here are the steps to generate the model. + +```bash + $ pip install onnx onnxruntime torch torchvision onnxscript + $ git clone --shallow-since=2025-07-1 https://github.com/pytorch/tutorials.git + $ cd tutorials && git reset --hard `git rev-list --max-parents=0 HEAD` + #update opset_version to 18 in advanced_source/super_resolution_with_onnxruntime.py + $ python3 advanced_source/super_resolution_with_onnxruntime.py + # copy the generated super_resolution.onnx and super_resolution.onnx.data to deepstream-ipc-test-sr. + $ cp advanced_source/super_resolution.onnx* /path/to/your/deepstream-ipc-test-sr +``` + +## Run + +Run with the command line. This sample act as either server or client based on command line arguments. + +```shell + # server + $ ./deepstream-ipc-test-app server + # client + $ ./deepstream-ipc-test-app client +``` +e.g. + +- Server generates a url using a local file. Multiple clients play the url. + +```shell + $ ./deepstream-ipc-test-app server file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 /tmp/test1 + $ ./deepstream-ipc-test-app client /tmp/test1 + $ ./deepstream-ipc-test-app client /tmp/test1 +``` + +- Server generates a url using RTSP. Client plays the url. + +```shell + $ ./deepstream-ipc-test-app server rtsp://127.0.0.1/video1 /tmp/test1 + $ ./deepstream-ipc-test-app client /tmp/test1 +``` + +- Server generates two urls. Client plays these two urls. + +```shell + $ ./deepstream-ipc-test-app server rtsp://127.0.0.1/video1 /tmp/test1 rtsp://127.0.0.1/video2 /tmp/test2 + $ ./deepstream-ipc-test-app client /tmp/test1 /tmp/test2 +``` + +The server accepts H.264/H.265 video stream RTSP URL and IPC socket path +as input. It does the decoding of the stream and listens for the connection +on the IPC socket path. It sends decoded data over IPC to the connected client. + +The client accepts IPC socket path as input. It sends connection request to the +server. Once server accepts the request, it starts receiving the decoded data +over IPC which is further pushed to deepstream pipeline. The rest of the pipeline +is similar to the deepstream-test3 sample. + +## Performance +### FPS Measurement +The client supports FPS statistic. + +```shell +$ IPC_SR_PERF_MODE=1 ./deepstream-ipc-test-app server \ + file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 /tmp/test1 \ + file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 /tmp/test2 \ + file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 /tmp/test3 \ + file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 /tmp/test4 + +$ IPC_SR_PERF_MODE=1 ./deepstream-ipc-test-app client \ + /tmp/test1 /tmp/test2 \ + /tmp/test3 /tmp/test4 | grep FPS +``` + +The client will output FPS in the terminal log, such as +``` +AVG FPS (num_sources 4 * 65.370): 261.481 +AVG FPS (num_sources 4 * 65.631): 262.525 +AVG FPS (num_sources 4 * 65.758): 263.031 +AVG FPS (num_sources 4 * 65.880): 263.518 +``` + +FPS statistic of the client: +|Device | FPS(batch_size:4) | +| ---------------- | ----- | +|A40 | 263.780 | +|Thor | 306.331 | + +### Latency Measurement +On the server side, Add `GstReferenceTimestampMeta` in the probe function of `nvunixfdsink` sink pad and serialize it through `serialize_meta`. +On the client side, Deserialize through `deserialize_meta`, and get the sending time in the probe function of the src pad of `nvunixfdsrc` + +Start sever with the following command. + +```shell +$ IPC_SR_PERF_MODE=1 ./deepstream-ipc-test-app server file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 /tmp/test1 +``` +Open another terminal, start client with the following command. + +```shell +$ IPC_SR_PERF_MODE=1 ./deepstream-ipc-test-app client /tmp/test1 | grep latency +``` + +The client will output latency in the terminal log, such as + +``` +IPC latency 0.406 +IPC latency 0.423 +IPC latency 0.429 +IPC latency 0.467 +IPC latency 0.402 +IPC latency 0.403 +IPC latency 0.402 +IPC latency 0.385 +``` +The latency of IPC is related to both CPU and GPU. + +|Device | Latency | +| ---------------- | --------- | +|A40 & AMD 7232P | ~0.4ms | +|Thor | ~0.117ms | + +NOTE: +- On Thor, it is a known issue that `Latency Measurement` is unavailable. +- To reuse engine files generated in previous runs, update the +model-engine-file parameter in the nvinfer config file to an existing +engine file. +- The engine model should be present to run IPC use-case. If it is not +present, the IPC test will timeout in first run as it takes some time +to generate the model. +- This example only support nvinfer. diff --git a/deepstream-ipc-test-sr/config_videotemplate.yml b/deepstream-ipc-test-sr/config_videotemplate.yml new file mode 100644 index 0000000..7473067 --- /dev/null +++ b/deepstream-ipc-test-sr/config_videotemplate.yml @@ -0,0 +1,29 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + #sr model output tensor width + width: 672 + #sr model output tensor height + height: 672 + diff --git a/deepstream-ipc-test-sr/deepstream_ipc_test_app.c b/deepstream-ipc-test-sr/deepstream_ipc_test_app.c new file mode 100644 index 0000000..736034f --- /dev/null +++ b/deepstream-ipc-test-sr/deepstream_ipc_test_app.c @@ -0,0 +1,854 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "gstnvdsmeta.h" +#include "nvds_yml_parser.h" +#include "gst-nvmessage.h" + +#include +#include +#include +#include +#include +#include +#include + +#define MAX_SOURCE_BINS 8 + +#define PGIE_CLASS_ID_VEHICLE 0 +#define PGIE_CLASS_ID_PERSON 2 + +/* By default, OSD process-mode is set to GPU_MODE. To change mode, set as: + * 0: CPU mode + * 1: GPU mode + */ +#define OSD_PROCESS_MODE 1 + +/* By default, OSD will not display text. To display text, change this to 1 */ +#define OSD_DISPLAY_TEXT 0 + +/* The muxer output resolution must be set if the input streams will be of + * different resolution. The muxer will scale all the input frames to this + * resolution. */ +#define MUXER_OUTPUT_WIDTH 224 +#define MUXER_OUTPUT_HEIGHT 224 + +/* width and height of model output tensor*/ +#define MODEL_OUTPUT_WIDTH 672 +#define MODEL_OUTPUT_HEIGHT 672 + +/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set + * based on the fastest source's framerate. */ +#define MUXER_BATCH_TIMEOUT_USEC 40000 + +#define TILED_OUTPUT_WIDTH 1280 +#define TILED_OUTPUT_HEIGHT 720 + +/* NVIDIA Decoder source pad memory feature. This feature signifies that source + * pads having this capability will push GstBuffers containing cuda buffers. */ +#define GST_CAPS_FEATURES_NVMM "memory:NVMM" + +typedef struct +{ + gchar *uri; + gchar *socket_path; + guint bus_id; + GstElement *pipeline; +} NvIpcServerPipeline; + +typedef struct +{ + gchar *socket_path[MAX_SOURCE_BINS]; + guint bus_id; + GstElement *pipeline; +} NvIpcClientPipeline; + +typedef struct +{ + GMainLoop *loop; + NvIpcServerPipeline ipcserver[MAX_SOURCE_BINS]; + NvIpcClientPipeline ipcclient; +} AppCtx; + +#define FPS_INTERVAL 300 + +static AppCtx gAppCtx = {0}; +static guint cintr = FALSE; +static gboolean g_perf_mode = FALSE; + +static gdouble get_current_timestamp() +{ + struct timeval t1; + double elapsed_time = 0; + gettimeofday(&t1, NULL); + elapsed_time = (t1.tv_sec) * 1000.0; + elapsed_time += (t1.tv_usec) / 1000.0; + return elapsed_time; +} + +static GstPadProbeReturn +server_sink_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, + gpointer u_data) +{ + if (g_perf_mode) { + GstBuffer *buf = (GstBuffer *) info->data; + if (gst_buffer_is_writable (buf)) { + GstCaps *caps = gst_caps_new_simple ("video/x-raw", + "server_send_time", G_TYPE_DOUBLE, get_current_timestamp(), NULL); + gst_buffer_add_reference_timestamp_meta (buf, caps, 0, 0); + gst_caps_unref(caps); + // g_print ("server_send_time %.3f %s\n", get_current_timestamp(), gst_caps_to_string (caps)); + } + } + return GST_PAD_PROBE_OK; +} + +static GstPadProbeReturn +client_source_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, + gpointer u_data) +{ + if (g_perf_mode) { + GstBuffer *buf = (GstBuffer *) info->data; + GstReferenceTimestampMeta *meta = + gst_buffer_get_reference_timestamp_meta (buf, NULL); + if (meta == NULL) { + // g_print ("%s: no reference timestamp meta\n", __FUNCTION__); + return GST_PAD_PROBE_OK; + } + GstCaps *caps = meta->reference; + double server_send_time = 0.0; + const GstStructure *str = gst_caps_get_structure (caps, 0); + gst_structure_get_double (str, "server_send_time", &server_send_time); + g_print ("IPC latency %.3f\n", get_current_timestamp() - server_send_time); + } + return GST_PAD_PROBE_OK; +} + +/* client_sgie_src_pad_buffer_probe will extract metadata received on OSD sink pad + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +client_sgie_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, + gpointer u_data) +{ + GstBuffer *buf = (GstBuffer *) info->data; + guint num_rects = 0; + NvDsObjectMeta *obj_meta = NULL; + guint vehicle_count = 0; + guint person_count = 0; + NvDsMetaList * l_frame = NULL; + NvDsMetaList * l_obj = NULL; + + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf); + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + num_rects = vehicle_count = person_count = 0; + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data); + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) { + obj_meta = (NvDsObjectMeta *) (l_obj->data); + if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) { + vehicle_count++; + num_rects++; + } + if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) { + person_count++; + num_rects++; + } + } + g_print ("Frame Number = %d Number of objects = %d " + "Vehicle Count = %d Person Count = %d\n", + frame_meta->frame_num, num_rects, vehicle_count, person_count); + } + return GST_PAD_PROBE_OK; +} + +static GstPadProbeReturn +client_osd_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data) +{ + static double start_time = 0.0f; + static int frame_count = 0; + if (frame_count == 0) { + start_time = get_current_timestamp(); + } else if (frame_count % FPS_INTERVAL == 0) { + int num_sources = *(guint *) u_data; + double current_time = get_current_timestamp(); + double fps = frame_count / (current_time - start_time) * 1000.0; + g_print ("AVG FPS (num_sources %d * %.3f): %.3f\n", + num_sources, fps, num_sources * fps); + } + frame_count++; + return GST_PAD_PROBE_OK; +} + +/** + * Function to handle program interrupt signal. + * It installs default handler after handling the interrupt. + */ +static void +_intr_handler(int signum) { + struct sigaction action; + + g_print("User Interrupted.. \n"); + + memset(&action, 0, sizeof(action)); + action.sa_handler = SIG_DFL; + + sigaction(SIGINT, &action, NULL); + + cintr = TRUE; +} + +/** + * Loop function to check the status of interrupts. + * It comes out of loop if application got interrupted. + */ +static gboolean +check_for_interrupt (gpointer data) +{ + if (cintr) { + cintr = FALSE; + g_main_loop_quit (gAppCtx.loop); + return FALSE; + } + return TRUE; +} + +/* +* Function to install custom handler for program interrupt signal. +*/ +static void +_intr_setup (void) +{ + struct sigaction action; + + memset (&action, 0, sizeof (action)); + action.sa_handler = _intr_handler; + + sigaction (SIGINT, &action, NULL); +} + +static gboolean +bus_call (GstBus * bus, GstMessage * msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + g_print ("End of stream\n"); + g_main_loop_quit (loop); + break; + case GST_MESSAGE_WARNING: + { + gchar *debug; + GError *error; + gst_message_parse_warning (msg, &error, &debug); + g_printerr ("WARNING from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + g_free (debug); + g_printerr ("Warning: %s\n", error->message); + g_error_free (error); + break; + } + case GST_MESSAGE_ERROR: + { + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + case GST_MESSAGE_ELEMENT: + { + if (gst_nvmessage_is_stream_eos (msg)) { + guint stream_id; + if (gst_nvmessage_parse_stream_eos (msg, &stream_id)) { + g_print ("Got EOS from stream %d\n", stream_id); + } + } + break; + } + case GST_MESSAGE_STATE_CHANGED: + { + GstState oldstate, newstate; + gst_message_parse_state_changed (msg, &oldstate, &newstate, NULL); + switch (newstate) { + case GST_STATE_PLAYING: + //g_print ("Pipeline running\n"); + break; + case GST_STATE_PAUSED: + if (oldstate == GST_STATE_PLAYING) { + //g_print ("Pipeline paused\n"); + } + break; + case GST_STATE_READY: + if (oldstate == GST_STATE_NULL) { + //g_print ("Pipeline ready\n"); + } else { + //g_print ("Pipeline stopped\n"); + } + break; + case GST_STATE_NULL: + //g_print ("Pipeline Null\n"); + g_main_loop_quit (loop); + return FALSE; + break; + default: + break; + } + break; + } + default: + break; + } + return TRUE; +} + +/* delete the pipeline */ +void +destroy_pipeline(AppCtx* appCtx) { + for(int i = 0; i < MAX_SOURCE_BINS; i++){ + if(appCtx->ipcserver[i].pipeline) { + gst_element_set_state (appCtx->ipcserver[i].pipeline, GST_STATE_NULL); + gst_object_unref (GST_OBJECT (appCtx->ipcserver[i].pipeline)); + g_source_remove (appCtx->ipcserver[i].bus_id); + g_print("server is closed uri: %s path: %s\n", + appCtx->ipcserver[i].uri, appCtx->ipcserver[i].socket_path); + g_free(appCtx->ipcserver[i].uri); + g_free(appCtx->ipcserver[i].socket_path); + } + } + if(appCtx->ipcclient.pipeline) { + GstBus *bus = NULL; + bus = gst_pipeline_get_bus (GST_PIPELINE (appCtx->ipcclient.pipeline)); + while (TRUE) { + GstMessage *message = gst_bus_pop (bus); + if (message == NULL) + break; + else if (GST_MESSAGE_TYPE (message) == GST_MESSAGE_ERROR) + bus_call (bus, message, appCtx->loop); + else + gst_message_unref (message); + } + gst_object_unref (bus); + gst_element_set_state (appCtx->ipcclient.pipeline, GST_STATE_NULL); + gst_object_unref (GST_OBJECT (appCtx->ipcclient.pipeline)); + g_source_remove (appCtx->ipcclient.bus_id); + for(int i = 0; i < MAX_SOURCE_BINS; i++){ + if(appCtx->ipcclient.socket_path[i]) { + g_print("client is closed path: %s\n", appCtx->ipcclient.socket_path[i]); + g_free(appCtx->ipcclient.socket_path[i]); + } + } + } + g_main_loop_unref (appCtx->loop); + g_print("destroy_pipeline end\n"); +} + +static void +cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data) +{ + GstCaps *caps = gst_pad_get_current_caps (decoder_src_pad); + if (!caps) { + caps = gst_pad_query_caps (decoder_src_pad, NULL); + } + const GstStructure *str = gst_caps_get_structure (caps, 0); + const gchar *name = gst_structure_get_name (str); + GstElement *source_bin = (GstElement *) data; + GstCapsFeatures *features = gst_caps_get_features (caps, 0); + + /* Need to check if the pad created by the decodebin is for video and not + * audio. */ + if (!strncmp (name, "video", 5)) { + /* Link the decodebin pad only if decodebin has picked nvidia + * decoder plugin nvdec_*. We do this by checking if the pad caps contain + * NVMM memory features. */ + if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) { + /* Get the source bin ghost pad */ + GstPad *bin_ghost_pad = gst_element_get_static_pad (source_bin, "src"); + if (!gst_ghost_pad_set_target (GST_GHOST_PAD (bin_ghost_pad), + decoder_src_pad)) { + g_printerr ("Failed to link decoder src pad to source bin ghost pad\n"); + } + gst_object_unref (bin_ghost_pad); + } else { + g_printerr ("Error: Decodebin did not pick nvidia decoder plugin.\n"); + } + } +} + +static void +decodebin_child_added (GstChildProxy * child_proxy, GObject * object, + gchar * name, gpointer user_data) +{ + g_print ("Decodebin child added: %s\n", name); + if (g_strrstr (name, "decodebin") == name) { + g_signal_connect (G_OBJECT (object), "child-added", + G_CALLBACK (decodebin_child_added), user_data); + } + if (g_strrstr (name, "source") == name) { + g_object_set(G_OBJECT(object),"drop-on-latency",true,NULL); + } +} + +static GstElement * +create_source_bin (guint index, gchar * uri) +{ + GstElement *bin = NULL, *uri_decode_bin = NULL; + gchar bin_name[16] = { 0 }; + + g_snprintf (bin_name, 15, "source-bin-%02d", index); + /* Create a source GstBin to abstract this bin's content from the rest of the + * pipeline */ + bin = gst_bin_new (bin_name); + + /* Source element for reading from the uri. + * We will use decodebin and let it figure out the container format of the + * stream and the codec and plug the appropriate demux and decode plugins. */ + uri_decode_bin = gst_element_factory_make ("nvurisrcbin", NULL); + g_object_set (G_OBJECT (uri_decode_bin), "file-loop", TRUE, NULL); + g_object_set (G_OBJECT (uri_decode_bin), "cudadec-memtype", 0, NULL); + + if (!bin || !uri_decode_bin) { + g_printerr ("One element in source bin could not be created.\n"); + return NULL; + } + + /* We set the input uri to the source element */ + g_object_set (G_OBJECT (uri_decode_bin), "uri", uri, NULL); + + /* Connect to the "pad-added" signal of the decodebin which generates a + * callback once a new pad for raw data has beed created by the decodebin */ + g_signal_connect (G_OBJECT (uri_decode_bin), "pad-added", + G_CALLBACK (cb_newpad), bin); + g_signal_connect (G_OBJECT (uri_decode_bin), "child-added", + G_CALLBACK (decodebin_child_added), bin); + + gst_bin_add (GST_BIN (bin), uri_decode_bin); + + /* We need to create a ghost pad for the source bin which will act as a proxy + * for the video decoder src pad. The ghost pad will not have a target right + * now. Once the decode bin creates the video decoder and generates the + * cb_newpad callback, we will set the ghost pad target to the video decoder + * src pad. */ + if (!gst_element_add_pad (bin, gst_ghost_pad_new_no_target ("src", + GST_PAD_SRC))) { + g_printerr ("Failed to add ghost pad in source bin\n"); + return NULL; + } + + return bin; +} + +static int +create_client_pipeline (int argc, char *argv[]) +{ + GMainLoop *loop = NULL; + GstElement *pipeline = NULL, *streammux = NULL, *sink = NULL, *pgie = NULL, *sgie = NULL, + *sr_conv = NULL, *sr_capsfilter = NULL, *sr_video_template = NULL, + *queue1, *queue2, *queue3, *queue4, *queue5, *nvvidconv = NULL, + *nvosd = NULL, *tiler = NULL; + GstCaps *caps = NULL; + GstBus *bus = NULL; + guint bus_watch_id; + GstPad *src_pad = NULL; + guint i = 0, num_sources = 0; + guint tiler_rows, tiler_columns; + guint pgie_batch_size; + gchar tmp_buf[256] = {0}; + gint tmp_buf_len = 255; + + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + gAppCtx.loop = loop = g_main_loop_new (NULL, FALSE); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + gAppCtx.ipcclient.pipeline = pipeline = gst_pipeline_new ("ipc-client-pipeline"); + + /* Create nvstreammux instance to form batches from one or more sources. */ + streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); + + if (!pipeline || !streammux) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), streammux); + + num_sources = argc - 2; + for (i = 0; i < num_sources; i++) { + GstElement *source = NULL, *caps_filter = NULL, *queue = NULL; + GstPad *sinkpad, *srcpad; + gchar pad_name[16] = { 0 }; + + const char *socket_path = argv[i + 2]; + gAppCtx.ipcclient.socket_path[i] = strdup(socket_path); + g_print("client is connected path: %s\n", gAppCtx.ipcclient.socket_path[i]); + + g_snprintf (tmp_buf, tmp_buf_len, "nvunixfdsrc_%u", i); + source = gst_element_factory_make ("nvunixfdsrc", tmp_buf); + gst_bin_add (GST_BIN (pipeline), source); + g_object_set (G_OBJECT(source), "socket-path", gAppCtx.ipcclient.socket_path[i], + "buffer_timestamp_copy", TRUE, NULL); + if (g_perf_mode) { + g_object_set (G_OBJECT(source), "meta-deserialization-lib", + "latency_serialization/liblatency_serialization.so", NULL); + } + + g_snprintf (tmp_buf, tmp_buf_len, "capsfilter_src_%u", i); + caps_filter = gst_element_factory_make ("capsfilter", NULL); + if (!caps_filter) { + g_printerr ("Failed to create caps_filter. Exiting.\n"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), caps_filter); + + caps = gst_caps_from_string ("video/x-raw(memory:NVMM),format=NV12"); + g_object_set (G_OBJECT(caps_filter), "caps", caps, NULL); + gst_caps_unref (caps); + + g_snprintf (tmp_buf, tmp_buf_len, "queue_src_%u", i); + queue = gst_element_factory_make ("queue", tmp_buf); + if (!queue) { + g_printerr ("Failed to create queue. Exiting.\n"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), queue); + /* link the elements together */ + if (!gst_element_link_many (source, caps_filter, queue, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + + src_pad = gst_element_get_static_pad (source, "src"); + if (!src_pad) + g_print ("Unable to get src pad\n"); + else + gst_pad_add_probe (src_pad, GST_PAD_PROBE_TYPE_BUFFER, + client_source_src_pad_buffer_probe, NULL, NULL); + gst_object_unref (src_pad); + + g_snprintf (pad_name, 15, "sink_%u", i); + sinkpad = gst_element_request_pad_simple (streammux, pad_name); + if (!sinkpad) { + g_printerr ("Streammux request sink pad failed. Exiting.\n"); + return -1; + } + + srcpad = gst_element_get_static_pad (queue, "src"); + if (!srcpad) { + g_printerr ("Failed to get src pad of source bin. Exiting.\n"); + return -1; + } + + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Failed to link source bin to stream muxer. Exiting.\n"); + return -1; + } + + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + } + + pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine"); + sgie = gst_element_factory_make ("nvinfer", "second-nvinference-engine"); + + /* Add queue elements between every two elements */ + queue1 = gst_element_factory_make ("queue", "queue1"); + queue2 = gst_element_factory_make ("queue", "queue2"); + queue3 = gst_element_factory_make ("queue", "queue3"); + queue4 = gst_element_factory_make ("queue", "queue4"); + queue5 = gst_element_factory_make ("queue", "queue5"); + sr_conv = gst_element_factory_make ("nvvideoconvert", "sr_conv"); + sr_capsfilter = gst_element_factory_make ("capsfilter", "sr_capsfilter"); + g_snprintf (tmp_buf, tmp_buf_len, "video/x-raw(memory:NVMM),format=NV12, width=%d, height=%d", + MODEL_OUTPUT_WIDTH, MODEL_OUTPUT_HEIGHT); + caps = gst_caps_from_string (tmp_buf); + g_object_set (G_OBJECT(sr_capsfilter), "caps", caps, NULL); + gst_caps_unref (caps); + sr_video_template = gst_element_factory_make("nvdsvideotemplate", "nvdsvideotemplate"); + g_object_set(G_OBJECT(sr_video_template), "customlib-name", "./video_template_impl/libnvds_vt_impl.so", NULL); + g_object_set(G_OBJECT(sr_video_template), "customlib-props", "config-file:config_videotemplate.yml", NULL); + + + /* Use nvtiler to composite the batched frames into a 2D tiled array based + * on the source of the frames. */ + tiler = gst_element_factory_make ("nvmultistreamtiler", "nvtiler"); + + /* Use convertor to convert from NV12 to RGBA as required by nvosd */ + nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); + + /* Create OSD to draw on the converted RGBA buffer */ + nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); + + if (g_perf_mode) { + sink = gst_element_factory_make ("fakesink", "nvvideo-renderer"); + } else if(prop.integrated) { + sink = gst_element_factory_make("nv3dsink", "nv3d-sink"); + } else { +#ifdef __aarch64__ + sink = gst_element_factory_make ("nv3dsink", "nvvideo-renderer"); +#else + sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer"); +#endif + } + + if (!pgie || !sgie || !tiler || !nvvidconv || !nvosd || !sink) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + + g_object_set (G_OBJECT (streammux), "batch-size", num_sources, NULL); + + g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", + MUXER_OUTPUT_HEIGHT, + "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); + + /* Configure the nvinfer element using the nvinfer config file. */ + g_object_set (G_OBJECT (pgie), + "config-file-path", "dsipctest_pgie_config.yml", "output-tensor-meta", TRUE, NULL); + g_object_set (G_OBJECT (sgie), + "config-file-path", "dsipctest_sgie_config.yml", NULL); + /* Override the batch-size set in the config file with the number of sources. */ + g_object_get (G_OBJECT (pgie), "batch-size", &pgie_batch_size, NULL); + if (pgie_batch_size != num_sources) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + pgie_batch_size, num_sources); + g_object_set (G_OBJECT (pgie), "batch-size", num_sources, NULL); + g_object_set (G_OBJECT (sgie), "batch-size", num_sources, NULL); + } + +#ifdef PLATFORM_TEGRA + /* for NvBufSurfaceMap in nvvideotemplate */ + g_object_set (G_OBJECT(sr_conv), "nvbuf-memory-type", 2, NULL); + g_object_set (G_OBJECT(sr_conv), "compute-hw", 1, NULL); + g_object_set (G_OBJECT(tiler), "compute-hw", 1, NULL); +#endif + + tiler_rows = (guint) sqrt (num_sources); + tiler_columns = (guint) ceil (1.0 * num_sources / tiler_rows); + /* we set the tiler properties here */ + g_object_set (G_OBJECT (tiler), "rows", tiler_rows, "columns", tiler_columns, + "width", TILED_OUTPUT_WIDTH, "height", TILED_OUTPUT_HEIGHT, NULL); + + g_object_set (G_OBJECT (nvosd), "process-mode", OSD_PROCESS_MODE, + "display-text", OSD_DISPLAY_TEXT, NULL); + + g_object_set (G_OBJECT (sink), "qos", 0, NULL); + // g_object_set (G_OBJECT (sink), "sync", FALSE, NULL); + g_object_set (G_OBJECT (streammux), "nvbuf-memory-type", 0, NULL); + + /* we add a message handler */ + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + gAppCtx.ipcclient.bus_id = bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); + gst_object_unref (bus); + + /* Set up the pipeline */ + /* we add all elements into the pipeline */ + gst_bin_add_many (GST_BIN (pipeline), queue1, pgie, + sr_conv, sr_capsfilter, sr_video_template, queue2, sgie, tiler, + queue3, nvvidconv, queue4, nvosd, queue5, sink, NULL); + + /* link the elements together */ + if (!gst_element_link_many (streammux, queue1, pgie, sr_conv, sr_capsfilter, sr_video_template, + queue2, sgie, tiler, queue3, nvvidconv, queue4, nvosd, queue5, sink, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the osd element, since by that time, the buffer would have + * had got all the metadata. */ + src_pad = gst_element_get_static_pad (sgie, "src"); + if (!src_pad) + g_print ("Unable to get src pad\n"); + else + gst_pad_add_probe (src_pad, GST_PAD_PROBE_TYPE_BUFFER, + client_sgie_src_pad_buffer_probe, NULL, NULL); + gst_object_unref (src_pad); + + src_pad = gst_element_get_static_pad (nvosd, "src"); + if (!src_pad) + g_print ("Unable to get src pad\n"); + else + gst_pad_add_probe (src_pad, GST_PAD_PROBE_TYPE_BUFFER, + client_osd_src_pad_buffer_probe, (gpointer)&num_sources, NULL); + gst_object_unref (src_pad); + + /* Set the pipeline to "playing" state */ + g_print ("Now playing:"); + for (i = 0; i < num_sources; i++) { + g_print (" %s,", argv[i + 2]); + } + g_print ("\n"); + gst_element_set_state (pipeline, GST_STATE_PLAYING); + + /* Wait till pipeline encounters an error or EOS */ + g_print ("Running...\n"); + g_main_loop_run (loop); + + g_print ("Deleting pipeline\n"); + destroy_pipeline(&gAppCtx); + return 0; +} + +static int +create_server_pipeline (int argc, char *argv[]) +{ + guint i =0, num_sources = 0; + GstPad *sink_pad = NULL; + num_sources = (argc - 2)/2; + GMainLoop *loop = NULL; + gAppCtx.loop = loop = g_main_loop_new (NULL, FALSE); + + for (i = 0; i < num_sources; i++) { + GstElement *pipeline = NULL; + GstBus *bus = NULL; + guint bus_watch_id; + GstElement *source_bin=NULL, *queue, *sink= NULL; + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + gAppCtx.ipcserver[i].pipeline = pipeline = gst_pipeline_new ("ipc-server-pipeline"); + if (!pipeline) { + g_printerr ("Failed to create pipeline. Exiting.\n"); + return -1; + } + + source_bin = create_source_bin (i, argv[(i*2) + 2]); + if (!source_bin) { + g_printerr ("Failed to create source bin. Exiting.\n"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), source_bin); + gAppCtx.ipcserver[i].uri = strdup(argv[(i*2) + 2]); + + queue = gst_element_factory_make ("queue", NULL); + if (!queue) { + g_printerr ("Failed to create queue. Exiting.\n"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), queue); + + sink = gst_element_factory_make ("nvunixfdsink", NULL); + if (!sink) { + g_printerr ("Failed to create nvunixfdsink. Exiting.\n"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), sink); + + gAppCtx.ipcserver[i].socket_path = strdup(argv[(i*2) + 3]); + g_print("server is started uri: %s path: %s\n", + gAppCtx.ipcserver[i].uri, gAppCtx.ipcserver[i].socket_path); + g_object_set (G_OBJECT(sink), "socket-path", gAppCtx.ipcserver[i].socket_path, + "buffer_timestamp_copy", TRUE, NULL); + if (g_perf_mode) { + g_object_set (G_OBJECT (sink), "sync", FALSE, + "meta-serialization-lib", "latency_serialization/liblatency_serialization.so", NULL); + } + + /* link the elements together */ + if (!gst_element_link_many (source_bin, queue, sink, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + + sink_pad = gst_element_get_static_pad (sink, "sink"); + if (!sink_pad) + g_print ("Unable to get sink pad\n"); + else + gst_pad_add_probe (sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + server_sink_sink_pad_buffer_probe, NULL, NULL); + gst_object_unref (sink_pad); + + /* we add a message handler */ + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + gAppCtx.ipcserver[i].bus_id = bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); + gst_object_unref (bus); + + gst_element_set_state (gAppCtx.ipcserver[i].pipeline, GST_STATE_PLAYING); + } + + /* Wait till pipeline encounters an error or EOS */ + g_print ("Running...\n"); + g_main_loop_run (loop); + + g_print ("Deleting pipeline\n"); + destroy_pipeline(&gAppCtx); + return 0; +} + +int +main (int argc, char *argv[]) +{ + int ret = 0; + + /* Check input arguments */ + if (argc < 3) { + g_printerr ("Usage: %s \n", argv[0]); + g_printerr ("OR: %s \n", argv[0]); + return -1; + } + + /* Standard GStreamer initialization */ + gst_init (&argc, &argv); + + /* setup signal handler */ + _intr_setup(); + g_timeout_add(400, check_for_interrupt, NULL); + + g_perf_mode = g_getenv("IPC_SR_PERF_MODE") && + !g_strcmp0(g_getenv("IPC_SR_PERF_MODE"), "1"); + g_print ("g_perf_mode: %d\n", g_perf_mode); + + if (strcmp(argv[1], "client") == 0 || strcmp(argv[1], "c") == 0) { + signal(SIGPIPE, SIG_IGN); + ret = create_client_pipeline(argc, argv); + } else if (strcmp(argv[1], "server") == 0 || strcmp(argv[1], "s") == 0) { + signal(SIGPIPE, SIG_IGN); + ret = create_server_pipeline(argc, argv); + } else { + g_printerr ("Invalid argument %s. Exiting.\n", argv[1]); + g_printerr ("Usage: %s \n", argv[0]); + g_printerr ("OR: %s \n", argv[0]); + return -1; + } + + return ret; +} diff --git a/deepstream-ipc-test-sr/dsipctest_pgie_config.yml b/deepstream-ipc-test-sr/dsipctest_pgie_config.yml new file mode 100644 index 0000000..a22c0e9 --- /dev/null +++ b/deepstream-ipc-test-sr/dsipctest_pgie_config.yml @@ -0,0 +1,55 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 0.0039215697906911373 + onnx-file: super_resolution.onnx + model-engine-file: super_resolution.onnx_b2_gpu0_fp16.engine + #model-engine-file: 1.engine + batch-size: 2 + network-mode: 2 + process-mode: 1 + model-color-format: 2 + interval: 0 + gie-unique-id: 1 + output-blob-names: output + cluster-mode: 2 + infer-dims: 1;224;224 + offsets: 0.0 + maintain-aspect-ratio: 0 + scaling-compute-hw: 1 + disable-output-host-copy: 1 + + ## 0=Detector, 1=Classifier, 2=Segmentation, 100=Other + network-type: 100 + # Enable tensor metadata output + output-tensor-meta: 1 + + #scaling-filter: 0 + #scaling-compute-hw: 0 + +class-attrs-all: + pre-cluster-threshold: 0.2 + topk: 20 + nms-iou-threshold: 0.5 diff --git a/deepstream-ipc-test-sr/dsipctest_sgie_config.yml b/deepstream-ipc-test-sr/dsipctest_sgie_config.yml new file mode 100644 index 0000000..06dfedc --- /dev/null +++ b/deepstream-ipc-test-sr/dsipctest_sgie_config.yml @@ -0,0 +1,44 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 0.00392156862745098 + onnx-file: /opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/resnet18_trafficcamnet_pruned.onnx + model-engine-file: /opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/resnet18_trafficcamnet_pruned.onnx_b2_gpu0_fp16.engine + labelfile-path: /opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/labels.txt + int8-calib-file: /opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/cal_trt.bin + batch-size: 2 + network-mode: 2 + num-detected-classes: 4 + interval: 0 + gie-unique-id: 1 + #scaling-filter=0 + scaling-compute-hw: 1 + cluster-mode: 2 + gie-unique-id: 2 + +class-attrs-all: + pre-cluster-threshold: 0.2 + topk: 20 + nms-iou-threshold: 0.5 diff --git a/deepstream-ipc-test-sr/latency_serialization/Makefile b/deepstream-ipc-test-sr/latency_serialization/Makefile new file mode 100644 index 0000000..a579182 --- /dev/null +++ b/deepstream-ipc-test-sr/latency_serialization/Makefile @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -) +CC:= gcc + +SRCS:= latency_serialization.c +LIB:=liblatency_serialization.so + +CFLAGS+= -fPIC + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + +LIBS += -shared -Wl,-no-undefined \ + -L$(LIB_INSTALL_DIR) \ + -Wl,-rpath,$(LIB_INSTALL_DIR) + +OBJS:= $(SRCS:.c=.o) +PKGS:= gstreamer-1.0 + +CFLAGS+=$(shell pkg-config --cflags $(PKGS)) +LIBS+=$(shell pkg-config --libs $(PKGS)) + +all: $(LIB) + +%.o: %.c $(INCS) + @echo $(CFLAGS) + $(CC) -c -o $@ $(CFLAGS) $< + +$(LIB): $(OBJS) + @echo $(CFLAGS) + $(CC) -o $@ $(OBJS) $(LIBS) + +install: $(LIB) + cp -rv $(LIB) $(LIB_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(LIB) diff --git a/deepstream-ipc-test-sr/latency_serialization/latency_serialization.c b/deepstream-ipc-test-sr/latency_serialization/latency_serialization.c new file mode 100644 index 0000000..176e234 --- /dev/null +++ b/deepstream-ipc-test-sr/latency_serialization/latency_serialization.c @@ -0,0 +1,53 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. + * All rights reserved. SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include + +void serialize_meta(GstBuffer *buf, guint8 **data, guint *len) { + if (buf == NULL || data == NULL || len == NULL) { + g_print("Invalid arguments\n"); + return; + } + guint out_len = 0; + GstReferenceTimestampMeta *meta = + gst_buffer_get_reference_timestamp_meta(buf, NULL); + if (meta == NULL) { + // g_print("serialize_meta: no reference timestamp meta\n"); + return; + } + GstCaps *ref = meta->reference; + if (ref) { + gchar *caps_str = gst_caps_to_string (ref); + out_len = strlen(caps_str) + 1; + // g_print("caps_str %s\n", caps_str); + *len = out_len; + // Allocate memory for the serialized data, free it after use + *data = g_malloc0(*len); + memcpy(*data, caps_str, out_len); + } else { + *data = NULL; + *len = 0; + } +} + +void deserialize_meta(GstBuffer *buf, guint8 *data, guint len) { + if (buf == NULL || data == NULL || len == 0) { + g_print("Invalid arguments\n"); + return; + } + GstCaps *caps = gst_caps_from_string((const gchar *)data); + if (caps) { + gst_buffer_add_reference_timestamp_meta(buf, caps, 0, 0); + gst_caps_unref(caps); + } + // g_print("deserialize_meta data: %s\n", (const gchar *)data); +} diff --git a/deepstream-ipc-test-sr/video_template_impl/Makefile b/deepstream-ipc-test-sr/video_template_impl/Makefile new file mode 100644 index 0000000..30a0fdf --- /dev/null +++ b/deepstream-ipc-test-sr/video_template_impl/Makefile @@ -0,0 +1,87 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -) +CXX:= g++ +NVCC:=/usr/local/cuda-$(CUDA_VER)/bin/nvcc +SRCS:= vt_impl.cpp yaml_parser.cpp data_conversion.cu + +INCS:= $(wildcard *.h) +LIB:=libnvds_vt_impl.so + +CFLAGS+= -I /usr/local/cuda/include \ + -I /opt/nvidia/deepstream/deepstream/sources/includes \ + -I /opt/nvidia/deepstream/deepstream/sources/gst-plugins/gst-nvdsvideotemplate/includes \ + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + +LIBS := -shared -Wl,-no-undefined -lnvds_yml_parser \ + -L/usr/local/cuda/lib64/ -lcudart -ldl -lpthread -lyaml-cpp \ + -L$(LIB_INSTALL_DIR) -lnvdsgst_helper -lnvdsgst_meta -lnvds_meta \ + -lnvbufsurface -lnvbufsurftransform -lnvdsbufferpool -lnvds_infer\ + -Wl,-rpath,$(LIB_INSTALL_DIR) + +ifeq ($(TARGET_DEVICE),aarch64) + LIBS += -L/usr/lib/aarch64-linux-gnu/tegra +endif + +# Gencode arguments +SMS ?= 75 80 86 89 90 100 120 +$(foreach sm,$(SMS),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm))) +ifeq ($(TARGET_DEVICE),aarch64) + GENCODE_FLAGS := +else + GENCODE_FLAGS := $(GENCODE_FLAGS) +endif + +OBJS:= $(SRCS:.cpp=.o) +OBJS:= $(OBJS:.cu=.o) + +PKGS:= gstreamer-1.0 gstreamer-base-1.0 gstreamer-video-1.0 + +CFLAGS+=$(shell pkg-config --cflags $(PKGS)) +LIBS+=$(shell pkg-config --libs $(PKGS)) +CFLAGS += -Wno-deprecated-declarations -fPIC + +all: $(LIB) + +%.o: %.cpp $(INCS) Makefile + @echo $(CFLAGS) + $(CXX) -c -o $@ $(CFLAGS) $< + +%.o: %.cu $(INCS) Makefile + @echo $(CFLAGS) + $(NVCC) -c -o $@ $(GENCODE_FLAGS) --compiler-options '-fPIC' $< + +$(LIB): $(OBJS) $(DEP) Makefile + @echo $(CFLAGS) + $(CXX) -o $@ $(OBJS) $(LIBS) + +$(DEP): $(DEP_FILES) + $(MAKE) -C customlib_impl/ + +install: $(LIB) + cp -rv $(LIB) $(LIB_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(LIB) diff --git a/deepstream-ipc-test-sr/video_template_impl/data_conversion.cu b/deepstream-ipc-test-sr/video_template_impl/data_conversion.cu new file mode 100644 index 0000000..f63838a --- /dev/null +++ b/deepstream-ipc-test-sr/video_template_impl/data_conversion.cu @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include +#include + +#define THREADS_PER_BLOCK 32 +#define THREADS_PER_BLOCK_1 (THREADS_PER_BLOCK - 1) + +__global__ void +Convert_FtFTensorKernel( + float *inBuffer, + unsigned char *outBuffer, + unsigned int width, + unsigned int height) +{ + unsigned int row = blockIdx.y * blockDim.y + threadIdx.y; + unsigned int col = blockIdx.x * blockDim.x + threadIdx.x; + + if (col < width && row < height) + { + int v = 255 * inBuffer[row * width + col]; + if(v < 0) { + v = 0; + } else if(v > 255) { + v = 255; + } + outBuffer[row * width + col] = v; + } +} + +void +Convert_FtFTensor( + float *inBuffer, + unsigned char *outBuffer, + unsigned int width, + unsigned int height) +{ + dim3 threadsPerBlock(THREADS_PER_BLOCK, THREADS_PER_BLOCK); + dim3 blocks((width+THREADS_PER_BLOCK_1)/threadsPerBlock.x, (height+THREADS_PER_BLOCK_1)/threadsPerBlock.y); + + Convert_FtFTensorKernel <<>> + (inBuffer, outBuffer, width, height); +} diff --git a/deepstream-ipc-test-sr/video_template_impl/data_conversion.h b/deepstream-ipc-test-sr/video_template_impl/data_conversion.h new file mode 100644 index 0000000..db50580 --- /dev/null +++ b/deepstream-ipc-test-sr/video_template_impl/data_conversion.h @@ -0,0 +1,33 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef __DATA_CONVERSION_H__ +#define __DATA_CONVERSION_H__ + +void +Convert_FtFTensor( + float *inBuffer, + unsigned char *outBuffer, + unsigned int width, + unsigned int height); + +#endif \ No newline at end of file diff --git a/deepstream-ipc-test-sr/video_template_impl/vt_impl.cpp b/deepstream-ipc-test-sr/video_template_impl/vt_impl.cpp new file mode 100644 index 0000000..ed84e5a --- /dev/null +++ b/deepstream-ipc-test-sr/video_template_impl/vt_impl.cpp @@ -0,0 +1,409 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvbufsurface.h" +#include "nvbufsurftransform.h" +#include "gst-nvquery.h" +#include "gstnvdsmeta.h" +#include "gst-nvevent.h" + +#include "nvdscustomlib_base.hpp" +#include +#include +#include +#include "yaml_parser.h" +#include "data_conversion.h" + +using namespace std; +using std::string; + +#define FORMAT_NV12 "NV12" +#define FORMAT_RGBA "RGBA" +#define HARDWARE_ACCELERATION 1 + +/* Strcture used to share between the threads */ +struct PacketInfo { + GstBuffer *inbuf; + guint frame_num; +}; + +class Algorithm : public DSCustomLibraryBase +{ +public: + Algorithm() { + outputthread_stopped = false; + m_cfgParams.m_tensor_width = 640; + m_cfgParams.m_tensor_height = 360; + m_pYbuf_cuda = NULL; + } + + /* Set Init Parameters */ + virtual bool SetInitParams(DSCustom_CreateParams *params); + + /* Set Custom Properties of the library */ + virtual bool SetProperty(Property &prop); + + /* Pass GST events to the library */ + virtual bool HandleEvent(GstEvent *event); + + virtual char *QueryProperties (); + + /* Process Incoming Buffer */ + virtual BufferResult ProcessBuffer(GstBuffer *inbuf); + + /* Retrun Compatible Caps */ + virtual GstCaps * GetCompatibleCaps (GstPadDirection direction, + GstCaps* in_caps, GstCaps* othercaps); + + /* Deinit members */ + ~Algorithm(); + +private: + /* Output Processing Thread, push buffer to downstream */ + void OutputThread(void); + +public: + guint source_id = 0; + guint m_frameNum = 0; + bool outputthread_stopped = false; + + /* Output Thread Pointer */ + std::thread *m_outputThread = NULL; + + /* Queue and Lock Management */ + std::queue m_processQ; + std::mutex m_processLock; + std::condition_variable m_processCV; + + /* Aysnc Stop Handling */ + gboolean m_stop = FALSE; + /*sr tensor width*/ + int m_tensor_width; + /*sr tensor height*/ + int m_tensor_height; + std::string m_config_file_path; + cfg_params m_cfgParams; + unsigned char* m_pYbuf_cuda; +}; + +// Create Custom Algorithm / Library Context +extern "C" IDSCustomLibrary *CreateCustomAlgoCtx(DSCustom_CreateParams *params) +{ + GST_DEBUG(" %d %s", __LINE__, __func__); + return new Algorithm(); +} + +// Set Init Parameters +bool Algorithm::SetInitParams(DSCustom_CreateParams *params) +{ + DSCustomLibraryBase::SetInitParams(params); + m_outputThread = new std::thread(&Algorithm::OutputThread, this); + GST_DEBUG(" %d %s", __LINE__, __func__); + + return true; +} + +// Return Compatible Output Caps based on input caps +GstCaps* Algorithm::GetCompatibleCaps (GstPadDirection direction, + GstCaps* in_caps, GstCaps* othercaps) +{ + GstCaps* result = NULL; + GstStructure *s1, *s2; + gint width, height; + gint i, num, denom; + const gchar *inputFmt = NULL; + + printf ("\n----------\ndirection = %d (1=Src, 2=Sink) -> %s:\nCAPS =" + " %s\n", direction, __func__, gst_caps_to_string(in_caps)); + printf ("%s : OTHERCAPS = %s\n", __func__, gst_caps_to_string(othercaps)); + + othercaps = gst_caps_truncate(othercaps); + othercaps = gst_caps_make_writable(othercaps); + + int num_output_caps = gst_caps_get_size (othercaps); + printf("num_output_caps:%d\n", num_output_caps); + num_output_caps = gst_caps_get_size (in_caps); + printf("in_caps, num_output_caps:%d\n", num_output_caps); + + // TODO: Currently it only takes first caps + s1 = gst_caps_get_structure(in_caps, 0); + for (i=0; i lk(m_processLock); + m_processCV.wait(lk, [&]{return m_processQ.empty();}); + m_stop = TRUE; + m_processCV.notify_all(); + lk.unlock(); + + /* Wait for OutputThread to complete */ + if (m_outputThread) { + m_outputThread->join(); + } + cudaFree(m_pYbuf_cuda); +} + +/* Process Buffer */ +BufferResult Algorithm::ProcessBuffer (GstBuffer *inbuf) +{ + GstMapInfo in_map_info; + + GST_DEBUG ("CustomLib: ---> Inside %s frame_num = %d\n", __func__, + m_frameNum++); + + // Push buffer to process thread for further processing + PacketInfo packetInfo; + packetInfo.inbuf = inbuf; + packetInfo.frame_num = m_frameNum; + + // Add custom preprocessing logic if required, here + // Pass the buffer to output_loop for further processing and pusing to next component + // Currently its just dumping few decoded video frames + + m_processLock.lock(); + m_processQ.push(packetInfo); + m_processCV.notify_all(); + m_processLock.unlock(); + + return BufferResult::Buffer_Async; +} + +void PostProcess_cuda(NvDsInferTensorMeta *meta, unsigned char* pY) { + for (unsigned int i = 0; i < meta->num_output_layers; i++) { + NvDsInferLayerInfo *info = &meta->output_layers_info[i]; + if (meta->out_buf_ptrs_dev[i]) { + int h = info->inferDims.d[1]; + int w = info->inferDims.d[2]; + Convert_FtFTensor((float*)meta->out_buf_ptrs_dev[i], pY, w, h); + cudaDeviceSynchronize(); + } + } +} + +/* replace lumin part */ +void replace_Y(unsigned char* pY, NvBufSurfaceParams *surParam) { + unsigned char* pSrc = pY; + int height = surParam->height; + int width = surParam->width; + int pitch = surParam->pitch; + unsigned char * dataPtr = (unsigned char *)surParam->dataPtr; + for (int i = 0; i < height; i++) { + cudaMemcpy (dataPtr, pSrc, width, cudaMemcpyDeviceToDevice); + pSrc += width; + dataPtr += pitch; + } +} + +/* Output Processing Thread */ +void Algorithm::OutputThread(void) +{ + GstFlowReturn flow_ret; + GstBuffer *outBuffer = NULL; + NvBufSurface *outSurf = NULL; + NvDsBatchMeta *batch_meta = NULL; + GstMapInfo in_map_info; + std::unique_lock lk(m_processLock); + printf("in OutputThread\n"); + while(1){ + /* Wait if processing queue is empty. */ + if (m_processQ.empty()) { + if (m_stop == TRUE) { + break; + } + m_processCV.wait(lk); + continue; + } + + PacketInfo packetInfo = m_processQ.front(); + m_processQ.pop(); + + m_processCV.notify_all(); + lk.unlock(); + + NvBufSurface *in_surf = getNvBufSurface (packetInfo.inbuf); + batch_meta = gst_buffer_get_nvds_batch_meta (packetInfo.inbuf); + + NvDsMetaList * l_frame = NULL; + nvds_acquire_meta_lock (batch_meta); + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data); + for (NvDsMetaList * l_user = frame_meta->frame_user_meta_list; + l_user != NULL; l_user = l_user->next) { + NvDsUserMeta *user_meta = (NvDsUserMeta *) l_user->data; + if (user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META) + continue; + NvDsInferTensorMeta *meta = (NvDsInferTensorMeta *) user_meta->user_meta_data; + PostProcess_cuda(meta, m_pYbuf_cuda); + NvBufSurfaceParams *surParam = &(in_surf->surfaceList[frame_meta->batch_id]); + if (surParam->colorFormat == NVBUF_COLOR_FORMAT_NV12 || + surParam->colorFormat == NVBUF_COLOR_FORMAT_NV12_709 ) { + if(in_surf->memType == NVBUF_MEM_CUDA_DEVICE) + replace_Y(m_pYbuf_cuda, surParam); + } + } + } + + nvds_release_meta_lock (batch_meta); + + // Transform IP case + outSurf = in_surf; + outBuffer = packetInfo.inbuf; + + // Output buffer parameters checking + if (outSurf->numFilled != 0) + { + g_assert ((guint)m_outVideoInfo.width == outSurf->surfaceList->width); + g_assert ((guint)m_outVideoInfo.height == outSurf->surfaceList->height); + } + + flow_ret = gst_pad_push (GST_BASE_TRANSFORM_SRC_PAD (m_element), + outBuffer); + GST_DEBUG ("CustomLib: %s in_surf=%p, Pushing Frame %d to downstream..." + " flow_ret = %d TS=%" GST_TIME_FORMAT " \n", __func__, in_surf, + packetInfo.frame_num, flow_ret, + GST_TIME_ARGS(GST_BUFFER_PTS(outBuffer))); + + lk.lock(); + } + outputthread_stopped = true; + printf("exit OutputThread\n"); +} diff --git a/deepstream-ipc-test-sr/video_template_impl/yaml_parser.cpp b/deepstream-ipc-test-sr/video_template_impl/yaml_parser.cpp new file mode 100644 index 0000000..5da7b31 --- /dev/null +++ b/deepstream-ipc-test-sr/video_template_impl/yaml_parser.cpp @@ -0,0 +1,89 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "yaml_parser.h" +#include +#include +#include +#include +#include "cuda_runtime_api.h" + #include + +using std::endl; +using std::cout; + +static gboolean +gst_parse_props_yaml (const gchar * cfg_file_path, cfg_params& cfg_params) +{ + gboolean ret = FALSE; + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + if(!(configyml.size() > 0)) { + cout << "Can't open config file (" << cfg_file_path << ")" << endl; + } + for(YAML::const_iterator itr = configyml["property"].begin(); itr != configyml["property"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "width") { + cfg_params.m_tensor_width = itr->second.as(); + } else if (paramKey == "height") { + cfg_params.m_tensor_height = itr->second.as(); + } else { + std::string paramVal = itr->second.as(); + printf("not need %s\n", paramVal.c_str()); + } + } + + ret = TRUE; +done: + return ret; +} + +/* Parse nvinfer config file for context params. Returns FALSE in case of an error. */ +gboolean +gst_parse_context_params_yaml (const gchar * cfg_file_path, cfg_params& cfg_params) +{ + gboolean ret = FALSE; + + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + if(!(configyml.size() > 0)) { + cout << "Can't open config file (" << cfg_file_path << ")" << endl; + } + /* 'property' group is mandatory. */ + if(configyml["property"]) { + if (!gst_parse_props_yaml (cfg_file_path, cfg_params)) { + g_printerr ("Failed to parse group property\n"); + goto done; + } + } + else { + g_printerr ("Could not find group property\n"); + goto done; + } + ret = TRUE; + +done: + if (!ret) { + g_printerr ("** ERROR: <%s:%d>: failed\n", __func__, __LINE__); + } + return ret; +} diff --git a/deepstream-ipc-test-sr/video_template_impl/yaml_parser.h b/deepstream-ipc-test-sr/video_template_impl/yaml_parser.h new file mode 100644 index 0000000..f8b188f --- /dev/null +++ b/deepstream-ipc-test-sr/video_template_impl/yaml_parser.h @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef __YAML_PARSER_H__ +#define __YAML_PARSER_H__ +#include +#include + + /* +* parameters of calibrator +*/ +struct cfg_params{ + /* model tensor width*/ + int m_tensor_width; + /*sr model tensor height*/ + int m_tensor_height; +}; + +gboolean gst_parse_context_params_yaml (const gchar * cfg_file_path, cfg_params& cal_params); + +#endif \ No newline at end of file diff --git a/deepstream-masktracker/README.md b/deepstream-masktracker/README.md new file mode 100644 index 0000000..7108e32 --- /dev/null +++ b/deepstream-masktracker/README.md @@ -0,0 +1,101 @@ +# MaskTracker in DeepStream + +## Introduction +This sample application demonstrates using MaskTracker with DeepStream SDK. MaskTracker simultaneously performs multi-object tracking and segmentation using advanced vision foundation models such as Segment Anything Model 2 (SAM2). It uses SAM2 to visually track and segment targets across frames, while automatically adding and removing targets as needed. It stores visual features in previous frames in a memory bank and use them to localize targets in a new frame. For algorithm and setup details, please refer to [DeepStream MaskTracker Documentation](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvtracker.html#masktracker-developer-preview). + +## Prerequisites +Users need to install Ubuntu 24.04 and NVIDIA driver 570.133.20 on x86 with dGPUs supported by DeepStream. Jetson devices may not support running the entire SAM2 network due to resource limitation. Check [here](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_docker_containers.html#prerequisites) for DeepStream container setup. +1. Download the latest DeepStream container image from NGC (e.g., DS 9.0 in the example below) + ```bash + export DS_IMG_NAME="nvcr.io/nvidia/deepstream:9.0-triton-multiarch" + docker pull $DS_IMG_NAME + ``` + +2. Git clone `deepstream_tools` and the current `deepstream_reference_apps` repository to the host machine, and enter MaskTracker directory inside the repository. `deepstream_tools` contains a directory `sam2-onnx-tensorrt`, which will be used to convert SAM2 models for TensorRT inference later. + ```bash + # Clone both repositories into one folder + git clone https://github.com/NVIDIA-AI-IOT/deepstream_tools.git + git clone https://github.com/NVIDIA-AI-IOT/deepstream_reference_apps.git + cd deepstream_reference_apps/deepstream-masktracker + ``` + +3. Download NVIDIA pretrained `PeopleNet` for detection from [NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet/files?version=deployable_quantized_onnx_v2.6.3(e.g., PeopleNet v2.6.3 in the example below). + + ```bash + # current directory: deepstream_reference_apps/deepstream-masktracker + mkdir -p models/PeopleNet + cd models/PeopleNet + wget --no-check-certificate --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet/versions/deployable_quantized_onnx_v2.6.3/zip -O peoplenet_deployable_quantized_onnx_v2.6.3.zip + unzip peoplenet_deployable_quantized_onnx_v2.6.3.zip + ``` + + The model files are now stored in `PeopleNet` directory as + + ```bash + deepstream-masktracker + ├── configs + ├── streams + └── models + └── PeopleNet + ├── labels.txt + ├── resnet34_peoplenet.onnx + └── resnet34_peoplenet_int8.txt + ``` +## Running the Application + +Launch the container from current directory, and execute the MaskTracker pipeline inside the container. The current [config](configs/deepstream_app_source1.txt) requires users to run with a display because it uses EGL sink to visualize the overlay results. To run through ssh without display, please change `type=2` to `1` in group `[sink0]` in that file. Users can check [DeepStream sink group](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_ref_app_deepstream.html#sink-group) for the usage of each sink. + + +```bash +cd ../.. +sudo xhost + # give container access to display +# current directory: deepstream_reference_apps/deepstream-masktracker +docker run --gpus all -it --rm --net=host --privileged -v /tmp/.X11-unix:/tmp/.X11-unix -v $(pwd):/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-masktracker -v $(pwd)/../../deepstream_tools/sam2-onnx-tensorrt:/opt/nvidia/deepstream/deepstream/sources/tracker_ReID/sam2-onnx-tensorrt -v $(pwd)/../deepstream-tracker-3d/streams/Retail02_short.mp4:/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-masktracker/streams/Retail02_short.mp4 -v $(pwd)/../3d-bodypose-deepstream/streams/bodypose.mp4:/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-masktracker/streams/bodypose.mp4 -e DISPLAY=$DISPLAY $DS_IMG_NAME +``` + +Inside container, run the following commands. Please note that when `deepstream-app` is launched for the first time, it tries to create model engine files, which may take a couple minutes, depending on HW platforms. + +```bash +# Install prerequisites +cd /opt/nvidia/deepstream/deepstream/ +bash user_additional_install.sh + +# Download and convert SAM2 model +export TRACKER_MODEL_DIR="/opt/nvidia/deepstream/deepstream/samples/models/Tracker" +mkdir -p $TRACKER_MODEL_DIR +cd /opt/nvidia/deepstream/deepstream/sources/tracker_ReID/sam2-onnx-tensorrt +bash run.sh + +# Run MaskTracker pipeline +cd /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-masktracker/configs +mkdir -p track_results +deepstream-app -c deepstream_app_source1.txt +``` + +## Customizing the Video +To run MaskTracker on other videos, in `deepstream_app_source1.txt`, change `uri=file://../streams/bodypose.mp4` to the new video name. For example, to use the retail video, set `uri=file://../streams/Retail02_short.mp4`. + +## Output Retrieval and Visualization + +### DeepStream Direct Visualization +When the pipeline is launced, DeepStream shows the output video like below while processing the input video. Segmentation masks, bounding boxes and IDs are overlaid for each target. The result video is saved as `out.mp4`. Below shows sample video visualization. + +![sample MaskTracker results](figures/.bodypose_osd.gif) +![sample MaskTracker results](figures/.retail_osd.gif) + +### Metadata Processing +The KITTI results for boxes and IDs can be found in `track_results` folder. A file will be created for each frame in each stream, and the data format is defined below. + +| object Label | object Unique Id | blank | blank | blank | bbox left | bbox top | bbox right | bbox bottom | blank | blank | blank | blank | blank | blank | blank | confidence | visibility (N/A) | Foot Image Position X (N/A) | Foot Image Position Y (N/A) | +|--------------|------------------|-------|-------|-------|-----------|----------|------------|-------------|-------|-------|-------|-------|-------|-------|-------|-----------|-----------------------|-----------------------|-----------------------| +| string | long unsigned | float | int | float | float | float | float | float | float | float | float | float | float | float | float | float | float | float | float | + +Each frame is saved as `track_results/00_000_xxxxxx.txt`. Sample output of a frame is like below. Note visibility and foot position are not available for MaskTracker. +```txt +person 1 0.0 0 0.0 964.094116 263.143738 1177.927734 851.131775 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.980957 +person 2 0.0 0 0.0 1298.530762 234.894257 1637.269897 844.630981 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.966797 +person 0 0.0 0 0.0 614.902649 227.807709 911.121948 852.950439 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.981934 +... +``` + +The segmentation mask for each target is generated in the `mask_params` field of `NvDsObjectMeta` in DeepStream meta data. As defined in `/opt/nvidia/deepstream/deepstream/sources/includes/nvll_osd_struct.h`, this data structure stores the segmentation mask as a float array with dimensions matching the target’s bounding box (rounded to integer values). In plugins or probes downstream to tracker, users can implement customized functions to access and store this data. diff --git a/deepstream-masktracker/configs/config_infer_primary.txt b/deepstream-masktracker/configs/config_infer_primary.txt new file mode 100644 index 0000000..826cf4f --- /dev/null +++ b/deepstream-masktracker/configs/config_infer_primary.txt @@ -0,0 +1,47 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 + +infer-dims=3;544;960 +int8-calib-file=../models/PeopleNet/resnet34_peoplenet_int8.txt +model-engine-file=../models/PeopleNet/resnet34_peoplenet.onnx_b1_gpu0_int8.engine +labelfile-path=../models/PeopleNet/labels.txt +onnx-file=../models/PeopleNet/resnet34_peoplenet.onnx + +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=1 +num-detected-classes=3 +interval=0 +gie-unique-id=1 +## 0=Group Rectangles, 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +cluster-mode=3 +#enable-dla=1 +#use-dla-core=0 +#scaling-filter=4 + +filter-out-class-ids=1;2 + +[class-attrs-all] +pre-cluster-threshold=0.1429 +nms-iou-threshold=0.4688 +minBoxes=3 +dbscan-min-score=0.7726 +eps=0.2538 +detected-min-w=10 +detected-min-h=10 diff --git a/deepstream-masktracker/configs/deepstream_app_source1.txt b/deepstream-masktracker/configs/deepstream_app_source1.txt new file mode 100644 index 0000000..8b2f037 --- /dev/null +++ b/deepstream-masktracker/configs/deepstream_app_source1.txt @@ -0,0 +1,108 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[application] +enable-perf-measurement=1 +perf-measurement-interval-sec=3 +kitti-track-output-dir=track_results + +[tiled-display] +enable=1 +rows=1 +columns=1 +width=1280 +height=720 +gpu-id=0 +nvbuf-memory-type=0 + +[source0] +enable=1 +type=3 +uri=file://../streams/bodypose.mp4 +# uri=file://../streams/Retail02_short.mp4 +num-sources=1 +gpu-id=0 +cudadec-memtype=0 + +[sink0] +enable=1 +type=2 +sync=0 +source-id=0 +gpu-id=0 +nvbuf-memory-type=0 +qos=0 + +[sink1] +enable=1 +type=3 +container=1 +codec=1 +enc-type=1 +sync=0 +bitrate=2000000 +profile=0 +output-file=out.mp4 +source-id=0 + +[osd] +enable=1 +gpu-id=0 +border-width=2 +text-size=15 +text-color=1;1;1;1; +text-bg-color=0.3;0.3;0.3;1 +font=Serif +show-clock=0 +clock-x-offset=800 +clock-y-offset=820 +clock-text-size=12 +clock-color=1;0;0;0 +nvbuf-memory-type=0 +display-mask=1 + +[streammux] +gpu-id=0 +live-source=0 +batch-size=1 +batched-push-timeout=-1 +width=1920 +height=1080 +enable-padding=0 +nvbuf-memory-type=0 + +[primary-gie] +enable=1 +gpu-id=0 +batch-size=1 +bbox-border-color0=1;0;0;1 +bbox-border-color1=0;1;1;1 +bbox-border-color2=0;0;1;1 +bbox-border-color3=0;1;0;1 +gie-unique-id=1 +nvbuf-memory-type=0 +interval=0 +config-file=config_infer_primary.txt + +[tracker] +enable=1 +tracker-width=1920 +tracker-height=1080 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_MaskTracker.yml +gpu-id=0 + +[tests] +file-loop=0 + diff --git a/deepstream-masktracker/figures/.bodypose_osd.gif b/deepstream-masktracker/figures/.bodypose_osd.gif new file mode 100644 index 0000000..1ffe040 Binary files /dev/null and b/deepstream-masktracker/figures/.bodypose_osd.gif differ diff --git a/deepstream-masktracker/figures/.retail_osd.gif b/deepstream-masktracker/figures/.retail_osd.gif new file mode 100644 index 0000000..d27528c Binary files /dev/null and b/deepstream-masktracker/figures/.retail_osd.gif differ diff --git a/deepstream-tracker-3d-multi-view/README.md b/deepstream-tracker-3d-multi-view/README.md new file mode 100644 index 0000000..073fef9 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/README.md @@ -0,0 +1,471 @@ +# Multi-View 3D Tracking in DeepStream + +MV3DT 12-camera live demo + +## Introduction + +This repository provides sample applications for Multi-View 3D Tracking (MV3DT) with DeepStream 9.0 SDK. MV3DT is a distributed, real-time multi-view multi-target 3D tracking framework built for large-scale, calibrated camera networks. It is designed to deliver robust object tracking and identity consistency across complex environments, leveraging camera calibration data as a prerequisite for accurate geometric reasoning. The sample applications support three detector models: [PeopleNet Transformer](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet_transformer_v2?version=deployable_v1.0), a general-purpose people detection transformer model, [PeopleNet v2.6.3](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet), a high-performance people detection model based on DetectNet_v2, and [RT-DETR 2D Warehouse](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/rtdetr_2d_warehouse?version=deployable_efficientvit_l2_v1.0), a real-time DETR model optimized for warehouse environments with multiple object classes. + +This repository aims to demonstrate MV3DT through live visualization of 3D tracking results, and is structured as follows: +- **[Prerequisites](#prerequisites)** - System requirements and setup instructions +- **[Option 1: Sample applications using DeepStream Container](#option-1-running-mv3dt-using-deepstream-container)** - The DeepStream Container has the DeepStream SDK pre-installed. The samples automate MV3DT config generation and launch the DeepStream app inside the container. +- **[Option 2: Sample applications using Inference Builder](#option-2-running-mv3dt-using-inference-builder)** - Inference Builder is an open-source tool that automates inference pipeline generation across AI frameworks and packages them as deployable containers. The samples in this repo are solely intended to demonstrate building and running MV3DT using Inference Builder. For additional capabilities, see the [Inference Builder README](https://github.com/NVIDIA-AI-IOT/inference_builder). +- **[Output Visualization Explanations](#output-visualization)** - Expected visualization from DeepStream On-Screen Display (OSD) and real-time Bird's Eye View (BEV) app +- **[Receiving 3D Tracking Metadata from Kafka](#receiving-3d-tracking-metadata-from-kafka)** - How to consume MV3DT tracking metadata from Kafka broker for downstream applications +- **[Customization](#customization)** - How to use MV3DT on custom datasets, and how to convert existing 2D DeepStream tracking pipelines to MV3DT pipeline + +As shown in the repo structure, MV3DT can be run using either DeepStream Container or Inference Builder. You can choose either approach to run the sample applications. We recommend starting with DeepStream Container for quick start and optionally trying out Inference Builder for advanced use cases, for example, integrating it with other AI frameworks or microservices. + +## Prerequisites +The sample applications in this repository require Ubuntu 24.04 and NVIDIA driver version 580.xx or higher; both x86 and Jetson platforms are supported. A graphical display server (e.g., X11) is required to view visualization results. If no physical display is available, a remote desktop solution such as VNC Viewer can be used as an alternative. + +#### Known Issues + +> On **Jetson Thor**, Option 2 Sample 2 (with Inference Builder, on 12-camera dataset) may hang due to file descriptor limitation in the third-party library libmosquitto. This issue is planned to be fixed in the next release. + +> On **DGX Spark**, the RT-DETR model requires TensorRT strongly-typed mode to produce valid inference outputs. Without it, detections may be missing and no bounding boxes will be shown. If you are running on DGX Spark with the RT-DETR detector, please add `strongly-typed=1` to the `[property]` section of `config_templates/config_pgie_rt_detr.txt` before launching the pipeline. + +> On **B200**, the sample apps with PeopleNetTransformer model may occasionally crash with a segmentation fault during inference. If this occurs, add `-e MALLOC_CHECK_=3` to the `docker run` command in launch scripts to mitigate the issue. + +#### Setup + +1. Please check [DeepStream Container Prerequisites](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_docker_containers.html#prerequisites) for DeepStream container setup. + +2. Git clone the current `deepstream_reference_apps` repository to the host machine and enter `deepstream-tracker-3d-multi-view` directory + ```bash + # Install Git LFS + sudo apt install git-lfs + git lfs install + + git clone https://github.com/NVIDIA-AI-IOT/deepstream_reference_apps.git + cd deepstream_reference_apps/deepstream-tracker-3d-multi-view + git lfs pull # In case repo is already cloned before installing git-lfs + ``` + +3. Run the automated prerequisites setup script: + + The setup script takes about 10-20 minutes to complete. And it may prompt you to enter your password for sudo access and github credentials. After the initial setup, if you close the session or restart your machine, you can still use this script to set up and check prerequisites, then the completion time will be much shorter. + + ```bash + # For DeepStream Container only (Option 1) + ./scripts/setup_prerequisites.sh + + # Additionally, if you want to try out Inference Builder (Option 2) + USE_INFERENCE_BUILDER=true ./scripts/setup_prerequisites.sh + + # [Expected output]: For both options, you should see all items checked with "✓" under "PREREQUISITES CHECK SUMMARY", and in the last line you should see: + # [SUCCESS] Prerequisites check passed! You're ready to use MV3DT. + ``` + + **Environment Variables:** + - `USE_INFERENCE_BUILDER` - Enable Inference Builder setup (default: false, DeepStream Container only) + - `BASE_DIR` - Base directory for Kafka and Inference Builder installations (default: `$HOME`) + - `DEEPSTREAM_IMAGE` - DeepStream Docker image (default: `nvcr.io/nvidia/deepstream:9.0-triton-multiarch` for x86 and Jetson platforms) + + * **Use case 1: If you want to use a different base directory** for Kafka and Inference Builder installations other than `$HOME`, you can set the `BASE_DIR` environment variable before running the script. + ```bash + export BASE_DIR=/path/to/your/preferred/base/directory + # If you want to use Inference Builder (Option 2), uncomment the lines below + # export USE_INFERENCE_BUILDER=true + # export INFERENCE_BUILDER_DIR="$BASE_DIR/inference_builder" + ./scripts/setup_prerequisites.sh + ``` + * **Use case 2: If you are on ARM SBSA platforms**, the DeepStream docker image will be different from the default one. Please set the `DEEPSTREAM_IMAGE` environment variable before running the script. + ```bash + export DEEPSTREAM_IMAGE=nvcr.io/nvidia/deepstream:9.0-triton-arm-sbsa + # If you want to use Inference Builder (Option 2), uncomment the line below + # export USE_INFERENCE_BUILDER=true + ./scripts/setup_prerequisites.sh + ``` + + + For manual setup, troubleshooting, or shutdown instructions for Kafka and Mqtt brokers, see: [Manual Setup Instructions](docs/manual-setup.md) + + +## Option 1: Running MV3DT using DeepStream Container + +The following examples demonstrate running MV3DT using DeepStream container. Note that the configurations are auto-generated from `config_templates` using the [auto-configurator](utils/README.md#deepstream_auto_configuratorpy). + +### Sample 1: 4-camera dataset +--- + +#### Quick Start +Run the provided script to quickly launch the 4-camera DeepStream pipeline. + +```bash +cd + +sudo xhost + # give container access to display, only need to run once per session +# [expected output]: access control disabled, clients can connect from any host + +# chmod +x scripts/test_4cam_ds.sh +./scripts/test_4cam_ds.sh + +# To use RT-DETR detector instead of the default PeopleNetTransformer: +# DETECTOR_MODEL=RTDETR ./scripts/test_4cam_ds.sh + +# To use PeopleNet v2.6.3 detector: +# DETECTOR_MODEL=PeopleNet2.6.3 ./scripts/test_4cam_ds.sh +``` + +Two separate windows will be launched. One named **Bird-Eye View of Multi-View 3D Tracking**, and the other named **DeepStreamTest5App**. You may need to toggle, arrange, or resize the windows to see both views. If anything goes wrong or the windows are not showing, please follow the step-by-step instructions below; otherwise, the quick-start script covers the same processes. + +**Note 1 (Important): When the script is launched for the first time, it tries to create model engine files use by MV3DT, which may take about 15 minutes, depending on HW platforms.** This process is only needed once for each dataset, and subsequent runs will use the generated engine files and launch immediately. + +**Note 2:** It's expected to see the following warnings. Those warnings will not affect the accuracy or performance of the pipeline. +- Load engine failed. Create engine again. +- INT8 calibration file not specified. Trying FP16 mode. +- GStreamer-WARNING + +**Window 1: Bird-Eye View of Multi-View 3D Tracking** +- This window shows the bird's-eye view of the multi-view 3D tracking results. It will show as blank map at the beginning. +- **The BEV visualization window will not exit automatically. To close the window, select the window and press 'q'.** + + +**Window 2: DeepStreamTest5App** +- This window shows the DeepStream on-screen display (OSD) of 4 camera views in a grid. This window is directly from Deepstream pipeline, and will show as black window at the beginning. +- To view a specific camera view in the DeepStream OSD window, left-click on the desired view. To return to the multi-camera grid view, simply right-click anywhere in the window. +- **The DeepStreamTest5App will exit automatically. To quit the DeepStreamTest5App window early, select the window and press 'q'.** After you press 'q', the app will terminate within a few seconds, and finally you will see "App run successful" printed. +- Note about window name: MV3DT is built upon DeepStreamTest5App with specific config files, which is why the window displays "DeepStreamTest5App" as its title. This base application provides essential IoT protocol support (Kafka and MQTT) required by MV3DT. Currently, the window name cannot be changed. For more details, see the [DeepStreamTest5App documentation](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_ref_app_test5.html). + + +**Expected output during engine generation period from window overview/spread:** + +Screenshot + +**Expected output after pipeline starts:** + +Screenshot + +#### Step-by-step Instructions +For detailed step-by-step instructions, see [DeepStream Container: Step-by-step Instructions](docs/step-by-step-deepstream.md#sample-1-4-camera-dataset). + +### Sample 2: 12-camera dataset +--- + +#### Quick Start +Run the provided script to quickly launch the 12-camera DeepStream pipeline. For detailed window explanations and important notes, see the [4-camera Quick Start section](#quick-start) above. + +```bash +# sudo xhost + # give container access to display + +# chmod +x scripts/test_12cam_ds.sh +./scripts/test_12cam_ds.sh + +# To use RT-DETR detector instead of the default PeopleNetTransformer: +# DETECTOR_MODEL=RTDETR ./scripts/test_12cam_ds.sh + +# To use PeopleNet v2.6.3 detector: +# DETECTOR_MODEL=PeopleNet2.6.3 ./scripts/test_12cam_ds.sh +``` +#### Step-by-step Instructions +For detailed step-by-step instructions, see [DeepStream Container: Step-by-step Instructions](docs/step-by-step-deepstream.md#sample-2-12-camera-dataset). + +## Option 2: Running MV3DT using Inference Builder + +Inference Builder is a tool that automatically generates inference pipelines and integrates them into either a microservice or a standalone application. In the samples in this repo, it is used to generate a Python package containing the MV3DT inference flow. + +Compared to traditional DeepStream configuration files, the Inference Builder configuration (e.g., `config_templates/ds_mv3dt.yaml`) is significantly simpler and more concise, making it easier to define and modify. + +### Sample 1: 4-camera dataset +--- + +#### Quick Start + +Run the provided script to quickly start the 4-camera DeepStream Inference Builder pipeline. +```bash +# sudo xhost + # give container access to display + +# If you changed BASE_DIR in the prerequisites setup, you need to export INFERENCE_BUILDER_DIR to the correct path +# export INFERENCE_BUILDER_DIR= + +# chmod +x scripts/test_4cam_ib.sh +./scripts/test_4cam_ib.sh + +# To use RT-DETR detector instead of the default PeopleNetTransformer: +# DETECTOR_MODEL=RTDETR ./scripts/test_4cam_ib.sh + +# To use PeopleNet v2.6.3 detector: +# DETECTOR_MODEL=PeopleNet2.6.3 ./scripts/test_4cam_ib.sh +``` + +For detailed window explanations and important notes, see the [4-camera DeepStream Quick Start section](#quick-start) above. Additional notes: + + +**Note 1:** By default, the application waits **1000 seconds** if there is no data being streamed before exiting gracefully. It is expected to see 0 FPS printed during the 1000 seconds wait time. This waiting time is controlled by the `inference_timeout` parameter in `config_templates/ds_mv3dt.yaml`. To avoid this delay: +* Option 1: Pre-generate the engine files and place them in the correct directories before starting the pipeline. Then, set `inference_timeout` to a lower value (e.g., 30 seconds). +* Option 2: Use the default 1000-second timeout for the initial run to allow engine file creation. For later runs, reduce `inference_timeout` (e.g., to 30 seconds) so the container exits promptly after inference completes. + +**Note 2: To quit the app early**, run this command in a separate terminal to stop the container: +`docker ps -q --filter "ancestor=inference-builder-mv3dt" | xargs docker stop`. + +If you are not able to see camera view or BEV view, please follow the step-by-step instructions; otherwise, the quick-start script covers the same processes. + + +**Expected output from window overview/spread:** + +Screenshot + +#### Step-by-step Instructions +For detailed step-by-step instructions, see [Inference Builder: Step-by-step Instructions](docs/step-by-step-inference-builder.md#sample-1-4-camera-dataset). + + +### Sample 2: 12-camera dataset +--- + +#### Quick Start +Run the provided script to quickly start the 12-camera DeepStream Inference Builder pipeline. + +For detailed window explanations and important notes, see the [4-camera DeepStream Quick Start section](#quick-start) above. + +```bash +# sudo xhost + # give container access to display + +# If you changed BASE_DIR in the prerequisites setup, you need to export INFERENCE_BUILDER_DIR to the correct path +# export INFERENCE_BUILDER_DIR= + +# chmod +x scripts/test_12cam_ib.sh +./scripts/test_12cam_ib.sh + +# To use RT-DETR detector instead of the default PeopleNetTransformer: +# DETECTOR_MODEL=RTDETR ./scripts/test_12cam_ib.sh + +# To use PeopleNet v2.6.3 detector: +# DETECTOR_MODEL=PeopleNet2.6.3 ./scripts/test_12cam_ib.sh +``` + +#### Step-by-step Instructions +For detailed step-by-step instructions, see [Inference Builder: Step-by-step Instructions](docs/step-by-step-inference-builder.md#sample-2-12-camera-dataset). + + +## Output Visualization +Whether you use Option 1 (DeepStream Container) or Option 2 (Inference Builder), both approaches launch two windows with similar visualizations. This section explains and demonstrates the content of these visualization windows. + +### DeepStream Direct Visualization +--- +When the pipeline is launched, DeepStream shows the output video like below while processing the input video. In the example frames below, you can see that objects detected across different cameras are assigned globally consistent IDs. And both 2D and 3D bounding boxes are visualized for each tracked object. + +In the Inference Builder OSD window, object IDs are visible directly in the grid view. In the DeepStream Container OSD window, object IDs are only visible when viewing a single camera. To enter single camera view with object IDs, left-click on the desired camera view. And to return to the multi-camera grid view, simply right-click anywhere in the window. + +

+ Example 1: 4-cam dataset, with PeopleNetTransformer, Inference Builder OSD
+ Sample 4-camera cam-view tracking results +

+ +

+ Example 2: 12-cam dataset, with RT-DETR, Deepstream OSD
+ Sample 12-camera cam-view tracking results with RT-DETR
+ Note: a forklift is also detected in the 2nd camera view (row 1, column 2). +

+ +#### Disabling DeepStream Direct Visualization + +If you don't need on-screen display, you can disable it: + +* **For DeepStream Container:** + * Remove the `--enable-osd` option from `deepstream_auto_configurator.py` command in the quick-start script and run again. + +* **For Inference Builder:** + * Comment out the `render_config` section in `config_templates/ds_mv3dt.yaml` and run the quick-start script again. + + + +### Real-time BEV visualization of 3D metadata from Kafka +--- + +The `kafka_bev_visualizer.py` script provides real-time bird's-eye view (BEV) visualization of 3D tracking data streamed via Kafka. + +Note that the BEV visualization script should be launched before launching the MV3DT app. + +* Command: + ```bash + python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --show-ids \ + --average-multi-cam + ``` +* Expected outputs (left: 4-camera, right: 12-camera) +
+ Sample 4-camera BEV tracking results + Sample 12-camera BEV tracking results +
+ + +* Note that the BEV visualization shows fused tracks, where trajectories of the same object from multiple cameras are averaged into one. If you want to see the individual trajectories from every camera, simply remove the `--average-multi-cam` option from the command. The output will then look like this: +
+ Sample 4-camera BEV tracking results + Sample 12-camera BEV tracking results +
+ +# Receiving 3D Tracking Metadata from Kafka + +MV3DT streams tracking metadata (frame ID, sensor ID, object IDs, 3D bounding boxes, etc.) to a Kafka topic as protobuf messages. The `kafka_client.py` script demonstrates how to connect to the Kafka broker, deserialize the protobuf messages, and print them as JSON. For building downstream applications using MV3DT tracking metadata, this can be a reference implementation. + +```bash +source mv3dt_venv/bin/activate + +# Default: connects to localhost:9092, topic 'mv3dt' +python utils/kafka_client.py + +# Custom broker and topic +python utils/kafka_client.py --broker localhost:9092 --topic mv3dt +``` + +# Customization + +This section provides customization options for the MV3DT pipeline. If you are new to DeepStream and want to try MV3DT on your own dataset, see [Running MV3DT on Custom Datasets](#running-mv3dt-on-custom-datasets). If you already have a working 2D DeepStream tracking pipeline, see [Converting your Existing 2D DeepStream Tracking Pipeline to MV3DT](#converting-your-existing-2d-deepstream-tracking-pipeline-to-mv3dt) for simple transformation to multi-view 3D pipeline. + + + +## Running MV3DT on Custom Datasets + +**Requirements:** +- Multi-view video streams must be synchronized. +- All video streams must have the same resolution. +- Camera calibration parameters (projection matrices) must be available. + +### Steps +--- + +1. **Organize your dataset** with the following structure: + ``` + your_dataset/ + ├── videos/ + │ ├── camera1.mp4 + │ ├── camera2.mp4 + │ └── ... + ├── camInfo/ + │ ├── camera1.yml + │ ├── camera2.yml + │ └── ... + ├── map.png (optional, for BEV visualization) + └── transforms.yml (optional, for BEV visualization) + ``` + +2. **Create camera calibration files** following the format of `datasets/mtmc_4cam/camInfo/Warehouse_Synthetic_Cam001.yml`. Replace the `projectionMatrix_3x4_w2p` values with your camera's projection matrix. For more details about these files, please refer to the [Single-View 3D Tracking](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvtracker.html#single-view-3d-tracking) and [The 3x4 Camera Projection Matrix](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvtracker.html#the-3x4-camera-projection-matrix) sections of the DeepStream documentation. + +3. **Optional: BEV visualization setup** - Prepare a BEV map image and create a `transforms.yml` file specifying the projection matrix that maps world coordinates (in meters) to BEV image coordinates, following the sample format in `datasets/mtmc_4cam/transforms.yml`. + +4. **Generate configurations** using the auto-configurator. Refer to `scripts/test_4cam_ds.sh` for the exact python command and environment variables needed. + +5. **Launch the MV3DT pipeline** using your generated configs. Refer to `scripts/test_4cam_ds.sh` for the exact Docker command and environment variables needed. + + +### How It Works +--- +The auto-configurator generates a complete set of MV3DT config files tailored to your dataset: + +1. **Detects dataset structure** - Scans your `videos/` and `camInfo/` directories to determine camera count and calibration files +2. **Generates pipeline configuration** - Creates `config_deepstream.txt` with appropriate source URIs, batch sizes, and display grid layout based on detected cameras +3. **Creates inter-camera communication configs** - Generates `pub_sub_info_config_0.yml` defining camera neighbor relationships for multi-view data sharing +4. **(Optional) Applies overrides** - Uses override files to customize tracker settings for specific datasets or requirements + +For detailed usage and all available options, see the **[Auto-Configurator Documentation](utils/README.md)**. + + +## Converting your Existing 2D DeepStream Tracking Pipeline to MV3DT +If you have an existing 2D detection and tracking pipeline using DeepStream, the auto-configurator can automatically enhance it to support multi-view 3D tracking, provided you have camera calibration files available. + +### Steps +--- + +1. **Organize your dataset** as in the previous [Running MV3DT on Custom Datasets](#running-mv3dt-on-custom-datasets) section. + +2. **Create camera calibration files** as in the previous section. + +3. **Optional: BEV visualization setup** (required if you want to enable BEV visualization) + - Follow the instructions in the previous section. + +4. **Optional: Set up your model configs** (required if using a different detector than PeopleNetTransformer) + - For example, if you are using PeopleNet as your detector, create a `PeopleNet` folder under `models` and modify `config_templates/config_pgie.txt` to point to your model files. + +5. **Optional: Set up your DeepStream pipeline config** (required if using a custom DeepStream pipeline config) + - Modify `config_templates/config_deepstream.txt` based on your use case. For example, if you have a custom `[pre-process]` section, copy that section to `config_templates/config_deepstream.txt`. Note that `[source%d]` and `[sink%d]` sections will be handled by the auto-configurator. + +6. **Place your tracker config file in the `config_templates` folder.** + + +7. **Generate MV3DT configs** + - Use the auto-configurator with `--tracker-config` argument, i.e. suppose your tracker config file is named `config_tracker_custom_2d.yml`, pass `--tracker-config=config_tracker_custom_2d.yml` to the auto-configurator. The auto-configurator assumes all template configs are located in the `config_templates` folder, so only the file name is needed, not the full path. + +8. **Launch the MV3DT pipeline** using your generated configs. + + +### A Step-by-step Example +--- + +1. To simulate a custom dataset, let's create a 6-camera subset (randomly selected) from the 12-camera dataset. Run the following command to generate a new dataset in `datasets/mtmc_6cam`. + ```bash + ./scripts/create_6cam_subset.sh + ``` + +2. Assume you have an existing 2D tracking config file. In this example, we will use the `config_tracker_2d.yml` file in the `config_templates` folder. + +3. Set up output directories and run the auto-configurator to generate the MV3DT config files. + ```bash + export DATASET_DIR=$PWD/datasets/mtmc_6cam/ + export EXPERIMENT_DIR=$PWD/experiments/deepstream/6cam + + mkdir -p $EXPERIMENT_DIR/infer-kitti-dump + mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump + + python utils/deepstream_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --tracker-config=config_tracker_2d.yml \ + --enable-msg-broker \ + --enable-osd \ + --output-dir=$EXPERIMENT_DIR + ``` + +4. Launch the MV3DT pipeline: + ```bash + export MODEL_REPO=$PWD/models + + docker run -t --privileged --rm --net=host --runtime=nvidia \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /workspace/experiments \ + nvcr.io/nvidia/deepstream:9.0-triton-multiarch \ + deepstream-test5-app -c config_deepstream.txt + ``` + +* For convenience, the same process is automated in the following scripts: + + ```bash + # Create the 6-camera dataset if not already done + # ./scripts/create_6cam_subset.sh + + # Option 1: using DeepStream Container + ./scripts/test_custom_2d_tracker_ds.sh + + # Option 2: using Inference Builder + ./scripts/test_custom_2d_tracker_ib.sh + ``` + +### How It Works +--- +The auto-configurator generates a complete set of MV3DT config files based on your custom 2D tracker, pipeline, and model configs, as well as your dataset. + +1. **Detects dataset structure** - Scans your `videos/` and `camInfo/` directories to determine camera count and calibration files +2. **Generates pipeline configuration** - Uses your updated `config_templates/config_deepstream.txt` as template and generates `$EXPERIMENT_DIR/config_deepstream.txt` with appropriate source URIs, batch sizes, and display grid layout +3. **Extends 2D tracker configs with MV3DT additional sections** - automatically injects MV3DT sections (`ObjectModelProjection` for 3D model projection, `MultiViewAssociator` for multi-view association, and `Communicator` for inter-camera communication) to your 2D tracker config +4. **Creates inter-camera communication configs** - Generates `$EXPERIMENT_DIR/pub_sub_info_config_0.yml` defining camera publish/subscribe relationships for multi-view data sharing +5. **(Optional) Applies overrides** - Uses override files to customize tracker settings for specific datasets or requirements + +For detailed usage and all available options, see the **[Auto-Configurator Documentation](utils/README.md)**. + + + + +## Python Util Scripts + +For more details on python utility scripts including auto-configuration generators and visualization tools, see +📁 **[Python Util Scripts Documentation](utils/README.md)** + + + + diff --git a/deepstream-tracker-3d-multi-view/assets/LICENSE b/deepstream-tracker-3d-multi-view/assets/LICENSE new file mode 100644 index 0000000..758d5be --- /dev/null +++ b/deepstream-tracker-3d-multi-view/assets/LICENSE @@ -0,0 +1,8 @@ +SPDX-FileCopyrightText: Copyright (c) 2018‑2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: CC-BY-ND-4.0 + +Licensed under the Creative Commons Attribution 4.0 International (CC BY 4.0) License. + +You may copy, distribute, modify, and build upon the material for any purpose, +including commercial use, as long as proper credit is given to the creator +and a link to the license is provided. \ No newline at end of file diff --git a/deepstream-tracker-3d-multi-view/assets/datasets.zip b/deepstream-tracker-3d-multi-view/assets/datasets.zip new file mode 100644 index 0000000..146b8de --- /dev/null +++ b/deepstream-tracker-3d-multi-view/assets/datasets.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5f84916042e7bf3126ebdc958b24f1f4824a09a99ea50428bef3963ed2a60aa +size 169275773 diff --git a/deepstream-tracker-3d-multi-view/config_templates/config_deepstream.txt b/deepstream-tracker-3d-multi-view/config_templates/config_deepstream.txt new file mode 100644 index 0000000..7b3d565 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/config_templates/config_deepstream.txt @@ -0,0 +1,140 @@ +[application] +kitti-track-output-dir=tracker-kitti-dump +gie-kitti-output-dir=infer-kitti-dump +perf-measurement-interval-sec=5 +enable-perf-measurement=1 + +[source0] +type=3 +enable=1 +cudadec-memtype=0 +gpu-id=0 +num-sources=1 +uri=file://videos/Warehouse_Synthetic_Cam001.mp4 + +[source1] +type=3 +enable=1 +cudadec-memtype=0 +gpu-id=0 +num-sources=1 +uri=file://videos/Warehouse_Synthetic_Cam002.mp4 + +[source2] +type=3 +enable=1 +cudadec-memtype=0 +gpu-id=0 +num-sources=1 +uri=file://videos/Warehouse_Synthetic_Cam003.mp4 + +[source3] +type=3 +enable=1 +cudadec-memtype=0 +gpu-id=0 +num-sources=1 +uri=file://videos/Warehouse_Synthetic_Cam004.mp4 + +[streammux] +enable-padding=0 +nvbuf-memory-type=0 +width=1920 +height=1080 +batched-push-timeout=-1 +batch-size=4 +live-source=0 +gpu-id=0 + +[primary-gie] +enable=1 +nvbuf-memory-type=0 +bbox-border-color3=0;1;0;1 +bbox-border-color2=0;0;1;1 +bbox-border-color1=0;1;1;1 +bbox-border-color0=1;0;0;1 +batch-size=4 +gpu-id=0 +gie-unique-id=1 +interval=0 +model-engine-file=/workspace/models/PeopleNetTransformer/peoplenet_transformer_model_op17.onnx_b4_gpu0_fp16.engine +config-file=config_pgie.txt + +[tracker] +enable=1 +tracker-width=1920 +tracker-height=1088 +display-tracking-id=1 +gpu-id=0 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +ll-config-file=config_tracker.yml + +[osd] +enable=1 +nvbuf-memory-type=0 +clock-color=1;0;0;0 +clock-text-size=12 +show-clock=0 +font=Serif +text-size=10 +text-bg-color=0.3;0.3;0.3;1 +text-color=1;1;1;1 +border-width=1 +clock-x-offset=800 +clock-y-offset=820 +gpu-id=0 +display-text=1 + +[tiled-display] +enable=1 +gpu-id=0 +width=1920 +height=1080 +nvbuf-memory-type=0 +rows=2 +columns=2 + +[sink0] +enable=1 +type=1 +qos=0 +gpu-id=0 +nvbuf-memory-type=0 +source-id=0 +sync=0 + +[sink1] +enable=0 +type=2 +qos=0 +gpu-id=0 +nvbuf-memory-type=0 +source-id=0 +sync=0 + +[sink2] +enable=0 +type=3 # file +container=1 # mp4 +codec=1 #h264 +enc-type=0 # hardware +sync=0 +# bitrate=2000000 +profile=0 # baseline +output-file=outVideos/tiled_display_raw.mp4 + +[sink3] +enable=0 +#Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker +type=6 +msg-conv-config=config_msgconv.txt +msg-conv-payload-type=2 +msg-conv-msg2p-new-api=0 +msg-conv-frame-interval=1 +msg-conv-msg2p-lib=/opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv_mega.so +msg-broker-proto-lib=/opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so +msg-broker-conn-str=localhost;9092;mv3dt +topic=mv3dt + +[tests] +file-loop=0 diff --git a/deepstream-tracker-3d-multi-view/config_templates/config_mqtt.txt b/deepstream-tracker-3d-multi-view/config_templates/config_mqtt.txt new file mode 100755 index 0000000..1cadee8 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/config_templates/config_mqtt.txt @@ -0,0 +1,13 @@ +[message-broker] +username = user +password = password +#client-id = uniqueID +#enable-tls = 1 +#tls-cafile = +#tls-capath = +#tls-certfile = +#tls-keyfile = +#share-connection = 1 +#loop-timeout = 2000 +#keep-alive = 60 +set-threaded = 0 \ No newline at end of file diff --git a/deepstream-tracker-3d-multi-view/config_templates/config_msgconv.txt b/deepstream-tracker-3d-multi-view/config_templates/config_msgconv.txt new file mode 100644 index 0000000..8a7a2d0 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/config_templates/config_msgconv.txt @@ -0,0 +1,21 @@ +[sensor1] +enable=1 +type=Camera +id=Camera1 + +[sensor2] +enable=1 +type=Camera +id=Camera2 + +[sensor3] +enable=1 +type=Camera +id=Camera3 + +[sensor4] +enable=1 +type=Camera +id=Camera4 + + diff --git a/deepstream-tracker-3d-multi-view/config_templates/config_pgie.txt b/deepstream-tracker-3d-multi-view/config_templates/config_pgie.txt new file mode 100644 index 0000000..f0a6f0c --- /dev/null +++ b/deepstream-tracker-3d-multi-view/config_templates/config_pgie.txt @@ -0,0 +1,23 @@ +[property] +gpu-id=0 +offsets=123.675;116.28;103.53 +net-scale-factor=0.0173520735728 +labelfile-path=/workspace/models/PeopleNetTransformer/detector_labels.txt +onnx-file=/workspace/models/PeopleNetTransformer/peoplenet_transformer_model_op17.onnx +tlt-model-key=nvidia_tao +batch-size=12 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=4 +filter-out-class-ids=0;2;3 +interval=0 +gie-unique-id=1 +output-blob-names=pred_boxes;pred_logits +infer-dims=3;544;960 +cluster-mode=4 +parse-bbox-func-name=NvDsInferParseCustomDDETRTAO +custom-lib-path=/workspace/models/PeopleNetTransformer/custom_parser/libnvds_infercustomparser_tao.so + +[class-attrs-all] +pre-cluster-threshold=0.3 +topk=20 diff --git a/deepstream-tracker-3d-multi-view/config_templates/config_pgie_peoplenet.txt b/deepstream-tracker-3d-multi-view/config_templates/config_pgie_peoplenet.txt new file mode 100644 index 0000000..38bfd99 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/config_templates/config_pgie_peoplenet.txt @@ -0,0 +1,14 @@ +[property] +net-scale-factor=0.0039215697906911373 +offsets=0.0;0.0;0.0 +labelfile-path=/workspace/models/PeopleNet2.6.3/detector_labels.txt +onnx-file=/workspace/models/PeopleNet2.6.3/resnet34_peoplenet.onnx +gie-unique-id=1 +network-type=0 +network-mode=2 +num-detected-classes=3 +filter-out-class-ids=1;2 +infer-dims=3;544;960 +model-color-format=0 +maintain-aspect-ratio=0 +output-tensor-meta=0 diff --git a/deepstream-tracker-3d-multi-view/config_templates/config_pgie_rt_detr.txt b/deepstream-tracker-3d-multi-view/config_templates/config_pgie_rt_detr.txt new file mode 100644 index 0000000..542c5e8 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/config_templates/config_pgie_rt_detr.txt @@ -0,0 +1,23 @@ +[property] +cluster-mode=4 +custom-lib-path=/workspace/models/RTDETR/custom_parser/libnvds_infercustomparser_tao.so +gie-unique-id=1 +infer-dims=3;544;960 +labelfile-path=/workspace/models/RTDETR/detector_labels.txt +maintain-aspect-ratio=1 +model-color-format=0 +net-scale-factor=0.00392156862745098 +network-mode=2 +network-type=0 +num-detected-classes=7 +filter-out-class-ids=6 +offsets=0;0;0 +onnx-file=/workspace/models/RTDETR/rtdetr_warehouse_v1.0.fp16.onnx +output-blob-names=pred_logits;pred_boxes +output-tensor-meta=1 +parse-bbox-func-name=NvDsInferParseCustomDDETRTAO +workspace-size=1048576 + +[class-attrs-all] +pre-cluster-threshold=0.4652309073592239 +topk=20 diff --git a/deepstream-tracker-3d-multi-view/config_templates/config_tracker.yml b/deepstream-tracker-3d-multi-view/config_templates/config_tracker.yml new file mode 100644 index 0000000..3c8fc11 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/config_templates/config_tracker.yml @@ -0,0 +1,107 @@ +%YAML:1.0 + +BaseConfig: + minDetectorConfidence: 0.027087304322979212 +TargetManagement: + enableBboxUnClipping: 1 + preserveStreamUpdateOrder: 0 + maxTargetsPerStream: 200 + minIouDiff4NewTarget: 0.22656630527418112 + minTrackerConfidence: 0.6957540479571296 + probationAge: 5 + maxShadowTrackingAge: 162 + earlyTerminationAge: 1 + maxTrajectoryBufferLength: -1 + outputTerminatedTracks: 0 +TrajectoryManagement: + useUniqueID: 0 + enableReAssoc: 1 + minMatchingScore4Overall: 0.9349462651721144 + minTrackletMatchingScore: 0.2940 + minMatchingScore4ReidSimilarity: 0 + matchingScoreWeight4TrackletSimilarity: 0.7981 + matchingScoreWeight4ReidSimilarity: 0 + minTrajectoryLength4Projection: 34 + prepLength4TrajectoryProjection: 58 + trajectoryProjectionLength: 33 + maxAngle4TrackletMatching: 67 + minSpeedSimilarity4TrackletMatching: 0.0574 + minBboxSizeSimilarity4TrackletMatching: 0.1013 + maxTrackletMatchingTimeSearchRange: 27 + trajectoryProjectionProcessNoiseScale: 0.0100 + trajectoryProjectionMeasurementNoiseScale: 100 + trackletSpacialSearchRegionScale: 0.0100 + reidExtractionInterval: 0 +DataAssociator: + dataAssociatorType: 0 + associationMatcherType: 1 + checkClassMatch: 0 + minMatchingScore4Overall: 0.4 + minMatchingScore4SizeSimilarity: 0.4 + minMatchingScore4Iou: 0.1393522182207021 + minMatchingScore4VisualSimilarity: 0.0520394823204932 + matchingScoreWeight4SizeSimilarity: 0.104589699500018 + matchingScoreWeight4Iou: 0.7844652139368062 + matchingScoreWeight4VisualSimilarity: 0.9294872869302965 + tentativeDetectorConfidence: 0.70167245554449 + minMatchingScore4TentativeIou: 0.1768733030811293 + minMatchingScore4PeerAssocIou: 0.2 +StateEstimator: + stateEstimatorType: 3 + processNoiseVar4Loc: 6497.75224242603 + processNoiseVar4Vel: 8035.858212054732 + measurementNoiseVar4Detector: 100.0000 + measurementNoiseVar4Tracker: 883.5847350922555 +ObjectModelProjection: + minPoseConfidence: 0.925 + outputFootLocation: 1 + outputVisibility: 1 + outputConvexHull: 0 + cameraModelFilepath: + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam001.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam002.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam003.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam004.yml + objectModelType: 0 +VisualTracker: + visualTrackerType: 2 + useColorNames: 1 + useHog: 1 + featureImgSizeLevel: 5 + featureFocusOffsetFactor_y: -0.10525549278780495 + filterLr: 0.025008995723548734 + filterChannelWeightsLr: 0.09460260509799209 + gaussianSigma: 0.44967391866177786 +MultiViewAssociator: + multiViewAssociatorType: 1 + enableLatePeerReAssoc: 1 + enableIDCorrection: 1 + enableSeeThrough: 1 + enableMsgSync: 1 + maxPeerTrackletSize: 50 + recentlyActiveAge: 178 + minCommonFrames4MatchScore: 2 + maxPeerToPredDistance4Fusion: 1.35 + minPeerVisibility4Fusion: 0.15 + minPeerTrackletMatchScore: 0.48 + maxTrackletMatchingTimeSearchRange: 1 + maxPeerFrameDiff4NoDet: 2 + communicatorInitSleepTime: 0 +Communicator: + communicatorType: 2 + pubSubInfoConfigPath: /workspace/experiments/pub_sub_info_config_0.yml + mqttProtoAdaptorConfigPath: /workspace/experiments/config_mqtt.txt +PoseEstimator: + poseEstimatorType: 1 + useVPICropScaler: 1 + batchSize: 1 + workspaceSize: 1000 + inferDims: [3, 256, 192] + networkMode: 1 + inputOrder: 0 + colorFormat: 0 + offsets: [123.6750, 116.2800, 103.5300] + netScaleFactor: 0.00392156 + onnxFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx + modelEngineFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx_b1_gpu0_fp16.engine + poseInferenceInterval: 30 \ No newline at end of file diff --git a/deepstream-tracker-3d-multi-view/config_templates/config_tracker_2d.yml b/deepstream-tracker-3d-multi-view/config_templates/config_tracker_2d.yml new file mode 100644 index 0000000..09c3c1f --- /dev/null +++ b/deepstream-tracker-3d-multi-view/config_templates/config_tracker_2d.yml @@ -0,0 +1,63 @@ +%YAML:1.0 + +BaseConfig: + minDetectorConfidence: 0.027087304322979212 +TargetManagement: + enableBboxUnClipping: 1 + preserveStreamUpdateOrder: 0 + maxTargetsPerStream: 200 + minIouDiff4NewTarget: 0.22656630527418112 + minTrackerConfidence: 0.6957540479571296 + probationAge: 1 + maxShadowTrackingAge: 162 + earlyTerminationAge: 1 + maxTrajectoryBufferLength: -1 + outputTerminatedTracks: 0 +TrajectoryManagement: + useUniqueID: 0 + enableReAssoc: 1 + minMatchingScore4Overall: 0.9349462651721144 + minTrackletMatchingScore: 0.2940 + minMatchingScore4ReidSimilarity: 0 + matchingScoreWeight4TrackletSimilarity: 0.7981 + matchingScoreWeight4ReidSimilarity: 0 + minTrajectoryLength4Projection: 34 + prepLength4TrajectoryProjection: 58 + trajectoryProjectionLength: 33 + maxAngle4TrackletMatching: 67 + minSpeedSimilarity4TrackletMatching: 0.0574 + minBboxSizeSimilarity4TrackletMatching: 0.1013 + maxTrackletMatchingTimeSearchRange: 27 + trajectoryProjectionProcessNoiseScale: 0.0100 + trajectoryProjectionMeasurementNoiseScale: 100 + trackletSpacialSearchRegionScale: 0.0100 + reidExtractionInterval: 0 +DataAssociator: + dataAssociatorType: 0 + associationMatcherType: 1 + checkClassMatch: 0 + minMatchingScore4Overall: 0.6671945991661751 + minMatchingScore4SizeSimilarity: 0.6718623956657859 + minMatchingScore4Iou: 0.1393522182207021 + minMatchingScore4VisualSimilarity: 0.0520394823204932 + matchingScoreWeight4SizeSimilarity: 0.104589699500018 + matchingScoreWeight4Iou: 0.7844652139368062 + matchingScoreWeight4VisualSimilarity: 0.9294872869302965 + tentativeDetectorConfidence: 0.70167245554449 + minMatchingScore4TentativeIou: 0.1768733030811293 + minMatchingScore4PeerAssocIou: 0.2 +StateEstimator: + stateEstimatorType: 1 + processNoiseVar4Loc: 6497.75224242603 + processNoiseVar4Vel: 8035.858212054732 + measurementNoiseVar4Detector: 100.0000 + measurementNoiseVar4Tracker: 883.5847350922555 +VisualTracker: + visualTrackerType: 1 + useColorNames: 1 + useHog: 1 + featureImgSizeLevel: 5 + featureFocusOffsetFactor_y: -0.10525549278780495 + filterLr: 0.025008995723548734 + filterChannelWeightsLr: 0.09460260509799209 + gaussianSigma: 0.44967391866177786 diff --git a/deepstream-tracker-3d-multi-view/config_templates/config_tracker_tuned_12cam.yml b/deepstream-tracker-3d-multi-view/config_templates/config_tracker_tuned_12cam.yml new file mode 100644 index 0000000..89290a8 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/config_templates/config_tracker_tuned_12cam.yml @@ -0,0 +1,112 @@ +%YAML:1.0 + +BaseConfig: + minDetectorConfidence: 0.294 +TargetManagement: + enableBboxUnClipping: 1 + preserveStreamUpdateOrder: 1 + maxTargetsPerStream: 200 + minIouDiff4NewTarget: 0.255 + minTrackerConfidence: 0.756 + probationAge: 3 + maxShadowTrackingAge: 150 + earlyTerminationAge: 2 + maxTrajectoryBufferLength: -1 + outputTerminatedTracks: 0 +TrajectoryManagement: + useUniqueID: 0 + enableReAssoc: 1 + minMatchingScore4Overall: 0.935 + minTrackletMatchingScore: 0.2940 + minMatchingScore4ReidSimilarity: 0 + matchingScoreWeight4TrackletSimilarity: 0.7981 + matchingScoreWeight4ReidSimilarity: 0 + minTrajectoryLength4Projection: 34 + prepLength4TrajectoryProjection: 58 + trajectoryProjectionLength: 33 + maxAngle4TrackletMatching: 67 + minSpeedSimilarity4TrackletMatching: 0.0574 + minBboxSizeSimilarity4TrackletMatching: 0.1013 + maxTrackletMatchingTimeSearchRange: 27 + trajectoryProjectionProcessNoiseScale: 0.0100 + trajectoryProjectionMeasurementNoiseScale: 100 + trackletSpacialSearchRegionScale: 0.0100 + reidExtractionInterval: 0 +DataAssociator: + dataAssociatorType: 0 + associationMatcherType: 1 + checkClassMatch: 1 + minMatchingScore4Overall: 0.586 + minMatchingScore4Iou: 0.253 + minMatchingScore4SizeSimilarity: 0.6 + minMatchingScore4VisualSimilarity: 0.6 + minMatchingScore4ReidSimilarity: 0.5 + matchingScoreWeight4Iou: 0.557 + matchingScoreWeight4SizeSimilarity: 0.326 + matchingScoreWeight4VisualSimilarity: 0.654 + matchingScoreWeight4ReidSimilarity: 0.1 + tentativeDetectorConfidence: 0.842 + minMatchingScore4TentativeIou: 0.364 + minMatchingScore4PeerAssocIou: 0.25 +StateEstimator: + stateEstimatorType: 3 + processNoiseVar4Loc: 3561.3 + processNoiseVar4Size: 447.7 + processNoiseVar4Vel: 3663.2 + measurementNoiseVar4Detector: 2078.3 + measurementNoiseVar4Tracker: 2108.0 + +ObjectModelProjection: + objectModelType: 0 + minPoseConfidence: 0.925 + outputFootLocation: 1 + outputVisibility: 1 + outputConvexHull: 0 + cameraModelFilepath: + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam001.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam002.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam003.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam004.yml +VisualTracker: + visualTrackerType: 2 + useColorNames: 1 + useHog: 1 + featureImgSizeLevel: 5 + featureFocusOffsetFactor_y: -0.10525549278780495 + filterLr: 0.025008995723548734 + filterChannelWeightsLr: 0.09460260509799209 + gaussianSigma: 0.44967391866177786 + +MultiViewAssociator: + multiViewAssociatorType: 1 + enableLatePeerReAssoc: 1 + enableIDCorrection: 1 + enableSeeThrough: 1 + enableMsgSync: 1 + maxPeerTrackletSize: 30 + recentlyActiveAge: 600 + minCommonFrames4MatchScore: 15 + minPeerTrackletMatchScore: 0.35 + minPeerVisibility4Fusion: 0.0213 + maxPeerToPredDistance4Fusion: 1.78 + maxTrackletMatchingTimeSearchRange: 1 + maxPeerFrameDiff4NoDet: 2 + communicatorInitSleepTime: 0 +Communicator: + communicatorType: 2 + pubSubInfoConfigPath: /workspace/experiments/pub_sub_info_config_0.yml + mqttProtoAdaptorConfigPath: /workspace/experiments/config_mqtt.txt +PoseEstimator: + poseEstimatorType: 1 + useVPICropScaler: 1 + batchSize: 1 + workspaceSize: 1000 + inferDims: [3, 256, 192] + networkMode: 1 + inputOrder: 0 + colorFormat: 0 + offsets: [123.6750, 116.2800, 103.5300] + netScaleFactor: 0.00392156 + onnxFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx + modelEngineFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx_b1_gpu0_fp16.engine + poseInferenceInterval: 29 \ No newline at end of file diff --git a/deepstream-tracker-3d-multi-view/config_templates/config_tracker_tuned_12cam_rt_detr.yml b/deepstream-tracker-3d-multi-view/config_templates/config_tracker_tuned_12cam_rt_detr.yml new file mode 100644 index 0000000..745c956 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/config_templates/config_tracker_tuned_12cam_rt_detr.yml @@ -0,0 +1,112 @@ +%YAML:1.0 + +BaseConfig: + minDetectorConfidence: 0.4249794671414514 +TargetManagement: + enableBboxUnClipping: 1 + preserveStreamUpdateOrder: 1 + maxTargetsPerStream: 200 + minIouDiff4NewTarget: 0.38243687471903276 + minTrackerConfidence: 0.741735665044422 + probationAge: 9 + maxShadowTrackingAge: 25 + earlyTerminationAge: 8 + maxTrajectoryBufferLength: -1 + outputTerminatedTracks: 0 +TrajectoryManagement: + useUniqueID: 0 + enableReAssoc: 1 + minMatchingScore4Overall: 0.935 + minTrackletMatchingScore: 0.294 + minMatchingScore4ReidSimilarity: 0 + matchingScoreWeight4TrackletSimilarity: 0.798 + matchingScoreWeight4ReidSimilarity: 0 + minTrajectoryLength4Projection: 34 + prepLength4TrajectoryProjection: 58 + trajectoryProjectionLength: 33 + maxAngle4TrackletMatching: 67 + minSpeedSimilarity4TrackletMatching: 0.057 + minBboxSizeSimilarity4TrackletMatching: 0.101 + maxTrackletMatchingTimeSearchRange: 27 + trajectoryProjectionProcessNoiseScale: 0.010 + trajectoryProjectionMeasurementNoiseScale: 100 + trackletSpacialSearchRegionScale: 0.010 + reidExtractionInterval: 0 +DataAssociator: + dataAssociatorType: 0 + associationMatcherType: 1 + checkClassMatch: 1 + minMatchingScore4Overall: 0.3302893671614782 + minMatchingScore4Iou: 0.24954038423337854 + minMatchingScore4SizeSimilarity: 0.7113775241492641 + minMatchingScore4VisualSimilarity: 0.3 + minMatchingScore4ReidSimilarity: 0.5 + matchingScoreWeight4Iou: 0.7917115752543623 + matchingScoreWeight4SizeSimilarity: 0.153194768090538 + matchingScoreWeight4VisualSimilarity: 0.9123026990374454 + matchingScoreWeight4ReidSimilarity: 0.1 + tentativeDetectorConfidence: 0.8847552837432789 + minMatchingScore4TentativeIou: 0.4499128093756385 + minMatchingScore4PeerAssocIou: 0.8239182971090612 +StateEstimator: + stateEstimatorType: 3 + processNoiseVar4Loc: 580.1268780450512 + processNoiseVar4Size: 4804.018474713682 + processNoiseVar4Vel: 435.8294162749178 + measurementNoiseVar4Detector: 9370.665057327513 + measurementNoiseVar4Tracker: 7510.009244259698 + +ObjectModelProjection: + objectModelType: 0 + minPoseConfidence: 0.9840686678221622 + outputFootLocation: 1 + outputVisibility: 1 + outputConvexHull: 0 + cameraModelFilepath: + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam001.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam002.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam003.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam004.yml +VisualTracker: + visualTrackerType: 2 + useColorNames: 1 + useHog: 1 + featureImgSizeLevel: 5 + featureFocusOffsetFactor_y: -0.10525549278780495 + filterLr: 0.025008995723548734 + filterChannelWeightsLr: 0.09460260509799209 + gaussianSigma: 0.44967391866177786 + +MultiViewAssociator: + multiViewAssociatorType: 1 + enableLatePeerReAssoc: 1 + enableIDCorrection: 0 + enableSeeThrough: 1 + enableMsgSync: 1 + maxPeerTrackletSize: 30 + recentlyActiveAge: 554 + minCommonFrames4MatchScore: 5 + minPeerTrackletMatchScore: 0.4979019568226545 + minPeerVisibility4Fusion: 0.39809962133314714 + maxPeerToPredDistance4Fusion: 1.5012866693280356 + maxTrackletMatchingTimeSearchRange: 1 + maxPeerFrameDiff4NoDet: 2 + communicatorInitSleepTime: 0 +Communicator: + communicatorType: 2 + pubSubInfoConfigPath: /workspace/experiments/pub_sub_info_config_0.yml + mqttProtoAdaptorConfigPath: /workspace/experiments/config_mqtt.txt +PoseEstimator: + poseEstimatorType: 1 + useVPICropScaler: 1 + batchSize: 1 + workspaceSize: 1000 + inferDims: [3, 256, 192] + networkMode: 1 + inputOrder: 0 + colorFormat: 0 + offsets: [123.6750, 116.2800, 103.5300] + netScaleFactor: 0.00392156 + onnxFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx + modelEngineFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx_b1_gpu0_fp16.engine + poseInferenceInterval: 30 \ No newline at end of file diff --git a/deepstream-tracker-3d-multi-view/config_templates/ds_mv3dt.yaml b/deepstream-tracker-3d-multi-view/config_templates/ds_mv3dt.yaml new file mode 100644 index 0000000..8017268 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/config_templates/ds_mv3dt.yaml @@ -0,0 +1,53 @@ +name: "deepstream-app" +model_repo: "/workspace/models" +models: +- name: PeopleNetTransformer + backend: deepstream/nvinfer + max_batch_size: 4 + input: + - name: media_url + data_type: TYPE_CUSTOM_BINARY_URLS + dims: [ -1 ] + optional: true + - name: mime + data_type: TYPE_CUSTOM_DS_MIME + dims: [ -1 ] + optional: true + - name: source_config + data_type: TYPE_CUSTOM_DS_SOURCE_CONFIG + dims: [ 1 ] + optional: true + output: + - name: output + data_type: TYPE_CUSTOM_DS_METADATA + dims: [ -1 ] + parameters: + infer_config_path: + - nvdsinfer_config.yaml + resize_video: [1080, 1920] + tracker_config: + ll_lib_file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + ll_config_file: /workspace/experiments/config_tracker.yml + width: 1920 + height: 1088 + display_tracking_id: true + msgbroker_config: + msgbroker_proto_lib_path: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + msgconv_config_path: /workspace/experiments/config_msgconv.txt + msgbroker_conn_str: localhost;9092;mv3dt + msgbroker_topic: mv3dt + msgconv_payload_type: 2 + msgconv_msg2p_new_api: 0 + msgconv_frame_interval: 1 + msgconv_msg2p_lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv_mega.so + perf_config: + enable_fps_logs: true + enable_latency_logs: true + kitti_output_path: + infer: /workspace/experiments/infer-kitti-dump/ + tracker: /workspace/experiments/tracker-kitti-dump/ + inference_timeout: 1000 + batch_timeout: -1 + render_config: + enable_display: true + enable_osd: true diff --git a/deepstream-tracker-3d-multi-view/config_templates/override_tracker_12cam.yml b/deepstream-tracker-3d-multi-view/config_templates/override_tracker_12cam.yml new file mode 100644 index 0000000..2ad8db6 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/config_templates/override_tracker_12cam.yml @@ -0,0 +1,15 @@ +MultiViewAssociator: + multiViewAssociatorType: 1 + enableLatePeerReAssoc: 1 + enableIDCorrection: 1 + enableSeeThrough: 1 + enableMsgSync: 1 + maxPeerTrackletSize: 50 + recentlyActiveAge: 178 + minCommonFrames4MatchScore: 2 + minPeerToPredDistance4Fusion: 1.35 + minPeerVisibility4Fusion: 0.15 + minPeerTrackletMatchScore: 0.48 + maxTrackletMatchingTimeSearchRange: 1 + maxPeerFrameDiff4NoDet: 2 + communicatorInitSleepTime: 0 \ No newline at end of file diff --git a/deepstream-tracker-3d-multi-view/config_templates/override_tracker_4cam.yml b/deepstream-tracker-3d-multi-view/config_templates/override_tracker_4cam.yml new file mode 100644 index 0000000..554fbc1 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/config_templates/override_tracker_4cam.yml @@ -0,0 +1,8 @@ +VisualTracker: + visualTrackerType: 0 +MultiViewAssociator: + multiViewAssociatorType: 1 + maxPeerToPredDistance4Fusion: 4.0 # mtmc 4.0 # 1.0 # sdg 1.3475168402461755 + minPeerTrackletMatchScore: 0.3 # mtmc 0.3 # sdg 0.48 # 0.48177346044415703 + minPeerVisibility4Fusion: 0.15 # sdg 0.1575362440818593 + recentlyActiveAge: 178 \ No newline at end of file diff --git a/deepstream-tracker-3d-multi-view/docs/manual-setup.md b/deepstream-tracker-3d-multi-view/docs/manual-setup.md new file mode 100644 index 0000000..ed98f20 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/docs/manual-setup.md @@ -0,0 +1,214 @@ +## Manual Setup Instructions + + +1. Please check [Deepstream Container Prerequisites](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_docker_containers.html#prerequisites) for Deepstream container setup, and download the latest DeepStream container image. + ```bash + docker pull nvcr.io/nvidia/deepstream:9.0-triton-multiarch + ``` +2. Git clone the current `deepstream_reference_apps` repository to the host machine and enter `deepstream-tracker-3d-multi-view` directory + ```bash + # Install Git LFS + sudo apt install git-lfs + git lfs install + + git clone https://github.com/NVIDIA-AI-IOT/deepstream_reference_apps.git + cd deepstream_reference_apps/deepstream-tracker-3d-multi-view + git lfs pull # In case repo is already cloned before installing git-lfs + ``` + +3. Unzip the datasets.zip managed by Git LFS + ```bash + unzip assets/datasets.zip + ``` + + +4. Download the `PeopleNetTransformer`, `RTDETR`, `PeopleNet v2.6.3`, and `BodyPose3DNet` models from NGC, and build custom parsers + * Download the models ([PeopleNetTransformer](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet_transformer_v2), [RT-DETR 2D Warehouse](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/rtdetr_2d_warehouse), [PeopleNet v2.6.3](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet), and [BodyPose3DNet](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/bodypose3dnet)) + ```bash + wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet_transformer_v2/deployable_v1.0/files?redirect=true&path=dino_fan_small_astro_delta.onnx' -O 'models/PeopleNetTransformer/peoplenet_transformer_model_op17.onnx' + wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/rtdetr_2d_warehouse/deployable_efficientvit_l2_v1.0/files?redirect=true&path=rtdetr_warehouse_v1.0.fp16.onnx' -O 'models/RTDETR/rtdetr_warehouse_v1.0.fp16.onnx' + wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/deployable_quantized_onnx_v2.6.3/files?redirect=true&path=resnet34_peoplenet.onnx' -O 'models/PeopleNet2.6.3/resnet34_peoplenet.onnx' + wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/bodypose3dnet/deployable_accuracy_onnx_1.0/files?redirect=true&path=bodypose3dnet_accuracy.onnx' -O 'models/BodyPose3DNet/bodypose3dnet_accuracy.onnx' + ``` + + * Build the custom parsers for both PeopleNetTransformer and RTDETR + ```bash + # Build PeopleNetTransformer custom parser + docker run --privileged --rm --net=host --runtime=nvidia \ + -v $PWD/models:/workspace/models \ + -w /workspace/models/PeopleNetTransformer \ + --entrypoint /bin/bash \ + nvcr.io/nvidia/deepstream:9.0-triton-multiarch \ + -c "cd custom_parser && make clean && make" + # [Expected output] You should see "libnvds_infercustomparser_tao.so" built under models/PeopleNetTransformer/custom_parser/. Warnings during build are expected. + + # Build RTDETR custom parser + docker run --privileged --rm --net=host --runtime=nvidia \ + -v $PWD/models:/workspace/models \ + -w /workspace/models/RTDETR \ + --entrypoint /bin/bash \ + nvcr.io/nvidia/deepstream:9.0-triton-multiarch \ + -c "cd custom_parser && make clean && make" + # [Expected output] You should see "libnvds_infercustomparser_tao.so" built under models/RTDETR/custom_parser/. Warnings during build are expected. + ``` + +5. Install and run the Mosquitto MQTT broker + + * Install Mosquitto and its client tools: + ```bash + sudo apt-add-repository ppa:mosquitto-dev/mosquitto-ppa + sudo apt update + sudo apt install mosquitto mosquitto-clients + ``` + + * Configure Mosquitto for optimal performance by enabling TCP_NODELAY: + ```bash + echo "set_tcp_nodelay true" | sudo tee /etc/mosquitto/conf.d/mv3dt.conf + ``` + + * After the installation, the Mosquitto broker service will be automatically started on port 1883. Restart it to apply the new config, then verify by running the provided test script. If the broker is active, you should see `Hello from Mosquitto test!` in the output: + ```bash + sudo systemctl restart mosquitto + + chmod +x ./scripts/mosquitto_test.sh + ./scripts/mosquitto_test.sh + ``` + + * If the previous step fails (e.g. seeing `Error: Connection refused`), use the following command to start Mosquitto on port 1883, and then run the test script again: + ```bash + mosquitto -p 1883 + # [Expected output] You should see "mosquitto version running" printed. + # You need to keep it running in a separate terminal window. To avoid this, you can use the following command to start it in the background: + # mosquitto -p 1883 -d + # [Expected output] Nothing will be printed. Use the mosquitto_test.sh script to verify the broker is running. + # And to kill it, you can use the following command: + # kill -9 $(lsof -t -i:1883) + ``` + + * Please refer to [Mosquitto documentation](https://mosquitto.org/download/) if you still encounter issues. + +6. Install and start a Kafka broker, and create a `mv3dt` topic: + * Follow the [Kafka quickstart](https://kafka.apache.org/quickstart) to download and start Kafka. The commands are provided below. **Note that please start a separate terminal window to keep the Kafka broker running.** + ```bash + # Kafka requires Java 17+. Check your Java version. + # If you see "Command 'java' not found" or it is older than 17, please install openjdk-17-jdk. + java -version + sudo apt install openjdk-17-jdk + + # Get Kafka + wget https://dlcdn.apache.org/kafka/4.2.0/kafka_2.13-4.2.0.tgz + tar -xzf kafka_2.13-4.2.0.tgz + cd kafka_2.13-4.2.0 + + # Start the Kafka environment + export KAFKA_CLUSTER_ID="$(bin/kafka-storage.sh random-uuid)" + + bin/kafka-storage.sh format --standalone -t $KAFKA_CLUSTER_ID -c config/server.properties + # [Expected output] You should see `Formatting dynamic metadata voter directory /tmp/kraft-combined-logs with metadata.version 4.0-IV3.` + + bin/kafka-server-start.sh config/server.properties + # [Expected output] You should see `Kafka Server started.` and it will keep logging `INFO` messages. + ``` + + * Create a `mv3dt` topic under broker server `localhost:9092`, and set the message retention to 30 seconds. + + ```bash + cd + + ./bin/kafka-topics.sh --bootstrap-server localhost:9092 \ + --create \ + --topic mv3dt \ + --partitions 1 \ + --replication-factor 1 \ + --config retention.ms=30000 \ + --if-not-exists + # [Expected output] Seeing `Created topic mv3dt.` or nothing if the topic already exists. + ``` + * After you have followed the above Kafka setup steps, in the future, you only need to run the following command to start Kafka: + ```bash + bin/kafka-server-start.sh config/server.properties + # [Expected output] It is expected to see DUPLICATE_BROKER_REGISTRATION in the logs. As long as the broker keeps running and logging INFO messages, you can proceed. + ``` + + * To stop a Kafka broker running in the background, you can use the following command: + ```bash + cd + bin/kafka-server-stop.sh + ``` + + +7. Install the required Python dependencies. Note that the scripts in this repo expect a virtual environment named `mv3dt_venv` located under the root of the repo. Please make sure to follow the following instructions exactly for quick start. + + ```bash + cd + + # Install required deb packages + sudo apt update + sudo apt install python3-tk python3.12-venv python3.12-dev + + # Create a python virtual enviornment named `mv3dt_venv` and install required python packages + python3 -m venv mv3dt_venv + source mv3dt_venv/bin/activate + + pip install -r requirements.txt + ``` + * Check the virtual environment. If any specific package fails, please install it manually with `pip install `. + ```bash + ls -d mv3dt_venv + # [Expected output] You should see "mv3dt_venv" printed. If you see "No such file or directory", please check the previous step "python3 -m venv mv3dt_venv". + + pip list + # [Expected output] You should see kafka-python, protobuf in the list + ``` + + +8. (Optional) This step is only needed if you choose to use Option 2: Inference Builder. + + Set up [Deepstream Inference Builder](https://github.com/NVIDIA-AI-IOT/inference_builder). It is recommended to clone the `inference_builder` repo outside of the current repo. + * Clone the inference builder repo + + ```bash + git clone https://github.com/NVIDIA-AI-IOT/inference_builder.git + cd inference_builder + git submodule update --init --recursive + ``` + * Create a new virtual environment for inference builder and install prerequisites. Please follow the following instructions exactly for quick start. **Note that there are 2 virtual environments used in this repo, `mv3dt_venv` and `ib_venv`. The scripts provided in the repo assumes that a `mv3dt_venv` folder is under the current repo, and a `ib_venv` folder is under the inference_builder repo.** + + ```bash + # Install required deb packages + sudo apt install protobuf-compiler + + # Deactivate the mv3dt_venv, and create a new virtual environment named ib_venv for inference builder + deactivate + python -m venv ib_venv + source ib_venv/bin/activate + pip3 install -r requirements.txt + ``` + * Check the virtual environment. If any specific package fails, please install it manually with `pip install `. + ```bash + ls -d ib_venv + # [Expected output] You should see "ib_venv" printed. If you see "No such file or directory", please check the previous step "python -m venv ib_venv". + + pip list + # [Expected output] You should see omegaconf 2.3.0 in the list + ``` + * Build a Docker image named `inference-builder-mv3dt:latest` with Inference Builder python dependencies. + ```bash + # Create a temporary Dockerfile + cat > ./Dockerfile.ib_mv3dt << 'EOF' + FROM nvcr.io/nvidia/deepstream:9.0-triton-multiarch + RUN pip3 install torch==2.7.0 omegaconf==2.3.0 + ENV GST_PLUGIN_PATH=/opt/nvidia/deepstream/deepstream/lib/gst-plugins + ENV LD_LIBRARY_PATH=/opt/nvidia/deepstream/deepstream/lib:$LD_LIBRARY_PATH + ENV NVSTREAMMUX_ADAPTIVE_BATCHING=yes + WORKDIR /mv3dt_app + EOF + + # [Expected output] You should see a Dockerfile.ib_mv3dt file created under the current directory. + + # Build the Docker image + docker build -f ./Dockerfile.ib_mv3dt -t inference-builder-mv3dt:latest . + + # [Expected output] You should see "naming to docker.io/library/inference-builder-mv3dt:latest" printed as the last line. + ``` + diff --git a/deepstream-tracker-3d-multi-view/docs/step-by-step-deepstream.md b/deepstream-tracker-3d-multi-view/docs/step-by-step-deepstream.md new file mode 100644 index 0000000..63ed6ca --- /dev/null +++ b/deepstream-tracker-3d-multi-view/docs/step-by-step-deepstream.md @@ -0,0 +1,96 @@ +# DeepStream Container: Step-by-step Instructions + +This page provides detailed step-by-step instructions for running MV3DT using the DeepStream Container. For quick start scripts, see the [main README](../README.md#option-1-running-mv3dt-using-deepstream-container). + +## Sample 1: 4-camera dataset + +1. Set up environment variables and prepare experiment directories + ```bash + export DATASET_DIR=$PWD/datasets/mtmc_4cam/ + export EXPERIMENT_DIR=$PWD/experiments/deepstream/4cam + export MODEL_REPO=$PWD/models + + mkdir -p $EXPERIMENT_DIR/infer-kitti-dump + mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump + mkdir -p $EXPERIMENT_DIR/outVideos + ``` +2. Generate DeepStream configuration files using the auto-configurator + + The auto-configurator automatically generates all necessary configuration files based on your dataset. It supports various output options (OSD display, video file output, Kafka streaming) and can work with both 2D and 3D tracker configurations. + + **About Override Files:** The `--config-overrides` parameter allows you to apply dataset-specific settings. For example, `override_tracker_4cam.yml` is optimized for the sample 4-camera dataset (which uses feet as world coordinate units). You can create custom override files for your own datasets. + + For more info on the auto-configurator, see [`utils/README.md`](../utils/README.md#deepstream_auto_configuratorpy). + + ```bash + # Activate the Python environment + source mv3dt_venv/bin/activate + + # Generate configs with 4-camera overrides + python utils/deepstream_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --enable-msg-broker \ + --enable-osd \ + --config-overrides=override_tracker_4cam.yml \ + --output-dir=$EXPERIMENT_DIR + + # [Expected output] You should see + # Generated files: + # - config_deepstream.txt (main pipeline config) + # - config_tracker.yml (3D tracker config) + # - config_msgconv.txt (message converter config) + # - pub_sub_info_config_0.yml (communication config) + + ``` +3. (Optional) Launch real-time BEV visualization + + Before launching the main MV3DT pipeline, optionally start the bird's-eye view visualizer to see real-time 3D tracking results. Please keep it running in a separate terminal window or add `&` to the end of the command to run it in the background. + + ```bash + # Start BEV visualization + python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids + + # [Expected output] You should see a window named "Bird-Eye View of Multi-View 3D Tracking" pop up and will display the live tracking results. + # Select the window and press 'q' to quit. + ``` + +4. Launch MV3DT + + The following command mounts the necessary folders into the DeepStream container and starts the `deepstream-test5-app` with MV3DT configs. + + ```bash + sudo xhost + # give container access to display + + docker run -t --privileged --rm --net=host --runtime=nvidia \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /workspace/experiments \ + nvcr.io/nvidia/deepstream:9.0-triton-multiarch \ + deepstream-test5-app -c config_deepstream.txt + + # [Expected output] You should see a window named "DeepStreamTest5App" pop up and will display 4 camera views in a grid. + # Select the window and press 'q' to quit early. + # The pipeline will quit automatically with "App run succesful" as the last line from the logs. + ``` + +## Sample 2: 12-camera dataset + +The steps are the same as for the 4-camera dataset, except setting `DATASET_DIR` and `EXPERIMENT_DIR` to the 12-camera directories. The auto-configurator automatically detects the number of cameras in your dataset and generates required config files for 12-camera dataset. + +```bash +export DATASET_DIR=$PWD/datasets/mtmc_12cam/ +export EXPERIMENT_DIR=$PWD/experiments/deepstream/12cam + +python utils/deepstream_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --enable-msg-broker \ + --enable-osd \ + --output-dir=$EXPERIMENT_DIR +``` diff --git a/deepstream-tracker-3d-multi-view/docs/step-by-step-inference-builder.md b/deepstream-tracker-3d-multi-view/docs/step-by-step-inference-builder.md new file mode 100644 index 0000000..cbbfef8 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/docs/step-by-step-inference-builder.md @@ -0,0 +1,104 @@ +# Inference Builder: Step-by-step Instructions + +This page provides detailed step-by-step instructions for running MV3DT using Inference Builder. For quick start scripts, see the [main README](../README.md#option-2-running-mv3dt-using-inference-builder). + +## Sample 1: 4-camera dataset + +1. Set up environment variables and prepare experiment directories + ```bash + export DATASET_DIR=$PWD/datasets/mtmc_4cam/ + export EXPERIMENT_DIR=$PWD/experiments/inference_builder/4cam + export MODEL_REPO=$PWD/models + + mkdir -p $EXPERIMENT_DIR/infer-kitti-dump + mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump + ``` + +2. Generate Inference Builder configuration files using the auto-configurator + + ```bash + # Activate the Python environment + source mv3dt_venv/bin/activate + + # Generate configs with 4-camera overrides + python utils/inference_builder_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --config-overrides=override_tracker_4cam.yml \ + --output-dir=$EXPERIMENT_DIR + # [Expected output] You should see + # Generated files: + # - ds_mv3dt.yaml (inference config with max_batch_size: 4) + # - config_tracker.yml (3D tracker config) + # - source_list_static.yaml (source configuration) + # - nvdsinfer_config.yaml (inference engine config with batch_size: 4) + # - config_msgconv.txt (message converter config) + # - pub_sub_info_config_0.yml (communication config) + + # Copy the generated nvdsinfer config to the model directory + cp $EXPERIMENT_DIR/nvdsinfer_config.yaml $MODEL_REPO/PeopleNetTransformer/ + ``` + +3. Generate a Python package at `$INFERENCE_BUILDER_DIR/builder/samples/mv3dt_app` containing the MV3DT inference flow. + ```bash + export INFERENCE_BUILDER_DIR= + cd $INFERENCE_BUILDER_DIR + source ib_venv/bin/activate + python builder/main.py $EXPERIMENT_DIR/ds_mv3dt.yaml \ + -o builder/samples/mv3dt_app \ + --server-type serverless + ``` + +4. (Optional) Launch real-time BEV visualization. + Please keep it running in a separate terminal window or add `&` to the end of the command to run it in the background. + + ```bash + # Return to the repo directory and activate the Python environment + cd + source mv3dt_venv/bin/activate + + # Start BEV visualization + python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids + + # [Expected output] You should see a window named "Bird-Eye View of Multi-View 3D Tracking" pop up and will display the live tracking results. + # Select the window and press 'q' to quit. + ``` + +5. Launch the `inference-builder-mv3dt:latest` container with volume mounts, including the Python package generated in the previous step. + + Note that this container is built during prerequisites setup. Please refer to the Inference Builder setup step in [Manual Setup Instructions](manual-setup.md) for more details. + ```bash + sudo xhost + # give container access to display + + docker run --privileged --rm -it --net=host --runtime=nvidia \ + -v $INFERENCE_BUILDER_DIR/builder/samples/mv3dt_app/deepstream-app:/mv3dt_app \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /mv3dt_app \ + inference-builder-mv3dt:latest \ + python3 __main__.py --source-config /workspace/experiments/source_list_static.yaml -s /dev/null + + # [Expected output] You should see a window named "python3" pop up and will display 4 camera views in a grid. + # Run this command to quit early: `docker ps -q --filter "ancestor=inference-builder-mv3dt" | xargs docker stop`. + # By default, the application waits up to **1000 seconds** if there is no data being streamed before exiting gracefully. You should see "Inference completed" as the last line from the logs. + ``` + + +## Sample 2: 12-camera dataset + +The steps are the same as for the 4-camera dataset, except setting `DATASET_DIR` and `EXPERIMENT_DIR` to the 12-camera directories. + +```bash +export DATASET_DIR=$PWD/datasets/mtmc_12cam/ +export EXPERIMENT_DIR=$PWD/experiments/inference_builder/12cam + +python utils/inference_builder_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --output-dir=$EXPERIMENT_DIR +``` diff --git a/deepstream-tracker-3d-multi-view/figures/12cam_bev.png b/deepstream-tracker-3d-multi-view/figures/12cam_bev.png new file mode 100644 index 0000000..3b5c271 Binary files /dev/null and b/deepstream-tracker-3d-multi-view/figures/12cam_bev.png differ diff --git a/deepstream-tracker-3d-multi-view/figures/12cam_bev_fused.png b/deepstream-tracker-3d-multi-view/figures/12cam_bev_fused.png new file mode 100644 index 0000000..006cf18 Binary files /dev/null and b/deepstream-tracker-3d-multi-view/figures/12cam_bev_fused.png differ diff --git a/deepstream-tracker-3d-multi-view/figures/12cam_osd.png b/deepstream-tracker-3d-multi-view/figures/12cam_osd.png new file mode 100644 index 0000000..05b1345 Binary files /dev/null and b/deepstream-tracker-3d-multi-view/figures/12cam_osd.png differ diff --git a/deepstream-tracker-3d-multi-view/figures/12cam_osd_rtdetr.png b/deepstream-tracker-3d-multi-view/figures/12cam_osd_rtdetr.png new file mode 100644 index 0000000..1487446 Binary files /dev/null and b/deepstream-tracker-3d-multi-view/figures/12cam_osd_rtdetr.png differ diff --git a/deepstream-tracker-3d-multi-view/figures/4cam_bev.png b/deepstream-tracker-3d-multi-view/figures/4cam_bev.png new file mode 100644 index 0000000..f3d19c0 Binary files /dev/null and b/deepstream-tracker-3d-multi-view/figures/4cam_bev.png differ diff --git a/deepstream-tracker-3d-multi-view/figures/4cam_bev_fused.png b/deepstream-tracker-3d-multi-view/figures/4cam_bev_fused.png new file mode 100644 index 0000000..ec08474 Binary files /dev/null and b/deepstream-tracker-3d-multi-view/figures/4cam_bev_fused.png differ diff --git a/deepstream-tracker-3d-multi-view/figures/4cam_osd.png b/deepstream-tracker-3d-multi-view/figures/4cam_osd.png new file mode 100644 index 0000000..2320b2c Binary files /dev/null and b/deepstream-tracker-3d-multi-view/figures/4cam_osd.png differ diff --git a/deepstream-tracker-3d-multi-view/figures/MV3DT_12cam_live.gif b/deepstream-tracker-3d-multi-view/figures/MV3DT_12cam_live.gif new file mode 100644 index 0000000..614a7d1 Binary files /dev/null and b/deepstream-tracker-3d-multi-view/figures/MV3DT_12cam_live.gif differ diff --git a/deepstream-tracker-3d-multi-view/figures/output_12cam.gif b/deepstream-tracker-3d-multi-view/figures/output_12cam.gif new file mode 100644 index 0000000..34230cc Binary files /dev/null and b/deepstream-tracker-3d-multi-view/figures/output_12cam.gif differ diff --git a/deepstream-tracker-3d-multi-view/figures/output_4cam.gif b/deepstream-tracker-3d-multi-view/figures/output_4cam.gif new file mode 100644 index 0000000..884f5a5 Binary files /dev/null and b/deepstream-tracker-3d-multi-view/figures/output_4cam.gif differ diff --git a/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ds.png b/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ds.png new file mode 100644 index 0000000..e7d76c9 Binary files /dev/null and b/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ds.png differ diff --git a/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ds_engine_generation.png b/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ds_engine_generation.png new file mode 100644 index 0000000..c70350b Binary files /dev/null and b/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ds_engine_generation.png differ diff --git a/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ib.png b/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ib.png new file mode 100644 index 0000000..0a6f306 Binary files /dev/null and b/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ib.png differ diff --git a/deepstream-tracker-3d-multi-view/models/PeopleNet2.6.3/detector_labels.txt b/deepstream-tracker-3d-multi-view/models/PeopleNet2.6.3/detector_labels.txt new file mode 100644 index 0000000..ef12c0a --- /dev/null +++ b/deepstream-tracker-3d-multi-view/models/PeopleNet2.6.3/detector_labels.txt @@ -0,0 +1,3 @@ +Person +Bag +Face diff --git a/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/custom_parser/Makefile b/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/custom_parser/Makefile new file mode 100644 index 0000000..18d8e1f --- /dev/null +++ b/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/custom_parser/Makefile @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + + +DS_VER = $(shell deepstream-app -v | awk '$$1~/DeepStreamSDK/ {print substr($$2,1,3)}' ) + +DS_SRC_PATH := /opt/nvidia/deepstream/deepstream-$(DS_VER) +CC:= g++ + +# Change to your deepstream SDK includes +CFLAGS+= -I$(DS_SRC_PATH)/sources/includes \ + -I/usr/local/cuda/include + +CFLAGS+= -Wall -std=c++11 -shared -fPIC + +LIBS+= -lnvinfer -L/usr/local/cuda/lib64 -lcudart -lcublas + +LFLAGS:= -Wl,--start-group $(LIBS) -Wl,--end-group + +SRCFILES:= nvdsinfer_custombboxparser_tao.cpp +TARGET_LIB:= libnvds_infercustomparser_tao.so + +all: $(TARGET_LIB) + +$(TARGET_LIB) : $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS) $(LFLAGS) + +clean: + rm -rf $(TARGET_LIB) diff --git a/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/custom_parser/nvdsinfer_custombboxparser_tao.cpp b/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/custom_parser/nvdsinfer_custombboxparser_tao.cpp new file mode 100644 index 0000000..bb07342 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/custom_parser/nvdsinfer_custombboxparser_tao.cpp @@ -0,0 +1,422 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include "nvdsinfer_custom_impl.h" +#include +#include +#include +#include + +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define MAX(a,b) ((a) > (b) ? (a) : (b)) +#define CLIP(a,min,max) (MAX(MIN(a, max), min)) +#define DIVIDE_AND_ROUND_UP(a, b) ((a + b - 1) / b) + +struct MrcnnRawDetection { + float y1, x1, y2, x2, class_id, score; +}; +/* This is a sample bounding box parsing function for the sample FasterRCNN + * + * detector model provided with the SDK. */ + +/* C-linkage to prevent name-mangling */ +extern "C" +bool NvDsInferParseCustomNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomBatchedNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomDDETRTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomEfficientDetTAO ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + + +extern "C" +bool NvDsInferParseCustomNMSTLT (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + if(outputLayersInfo.size() != 2) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 2 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + // Host memory for "nms" which has 2 output bindings: + // the order is bboxes and keep_count + float* out_nms = (float *) outputLayersInfo[0].buffer; + int * p_keep_count = (int *) outputLayersInfo[1].buffer; + const int out_class_size = detectionParams.numClassesConfigured; + const float threshold = detectionParams.perClassThreshold[0]; + + float* det; + + for (int i = 0; i < p_keep_count[0]; i++) { + det = out_nms + i * 7; + + // Output format for each detection is stored in the below order + // [image_id, label, confidence, xmin, ymin, xmax, ymax] + if ( det[2] < threshold) continue; + assert((int) det[1] < out_class_size); + +#if 0 + std::cout << "id/label/conf/ x/y x/y -- " + << det[0] << " " << det[1] << " " << det[2] << " " + << det[3] << " " << det[4] << " " << det[5] << " " << det[6] << std::endl; +#endif + NvDsInferObjectDetectionInfo object = {}; + object.classId = (int) det[1]; + object.detectionConfidence = det[2]; + + /* Clip object box co-ordinates to network resolution */ + object.left = CLIP(det[3] * networkInfo.width, 0, networkInfo.width - 1); + object.top = CLIP(det[4] * networkInfo.height, 0, networkInfo.height - 1); + object.width = CLIP((det[5] - det[3]) * networkInfo.width, 0, networkInfo.width - 1); + object.height = CLIP((det[6] - det[4]) * networkInfo.height, 0, networkInfo.height - 1); + + objectList.push_back(object); + } + + return true; +} + +extern "C" +bool NvDsInferParseCustomBatchedNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + + if(outputLayersInfo.size() != 4) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 4 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + /* Host memory for "BatchedNMS" + BatchedNMS has 4 output bindings, the order is: + keepCount, bboxes, scores, classes + */ + int* p_keep_count = (int *) outputLayersInfo[0].buffer; + float* p_bboxes = (float *) outputLayersInfo[1].buffer; + float* p_scores = (float *) outputLayersInfo[2].buffer; + float* p_classes = (float *) outputLayersInfo[3].buffer; + + const float threshold = detectionParams.perClassThreshold[0]; + + const int keep_top_k = 200; + const char* log_enable = std::getenv("ENABLE_DEBUG"); + + if(log_enable != NULL && std::stoi(log_enable)) { + std::cout <<"keep cout" + <= detectionParams.numClassesConfigured) continue; + if(p_bboxes[4*i+2] < p_bboxes[4*i] || p_bboxes[4*i+3] < p_bboxes[4*i+1]) continue; + + NvDsInferObjectDetectionInfo object = {}; + object.classId = (int) p_classes[i]; + object.detectionConfidence = p_scores[i]; + + /* Clip object box co-ordinates to network resolution */ + object.left = CLIP(p_bboxes[4*i] * networkInfo.width, 0, networkInfo.width - 1); + object.top = CLIP(p_bboxes[4*i+1] * networkInfo.height, 0, networkInfo.height - 1); + object.width = CLIP(p_bboxes[4*i+2] * networkInfo.width, 0, networkInfo.width - 1) - object.left; + object.height = CLIP(p_bboxes[4*i+3] * networkInfo.height, 0, networkInfo.height - 1) - object.top; + + if(object.height < 0 || object.width < 0) + continue; + objectList.push_back(object); + } + return true; +} + +extern "C" +bool NvDsInferParseCustomMrcnnTLTV2 (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + auto layerFinder = [&outputLayersInfo](const std::string &name) + -> const NvDsInferLayerInfo *{ + for (auto &layer : outputLayersInfo) { + if (layer.dataType == FLOAT && + (layer.layerName && name == layer.layerName)) { + return &layer; + } + } + return nullptr; + }; + + const NvDsInferLayerInfo *detectionLayer = layerFinder("generate_detections"); + const NvDsInferLayerInfo *maskLayer = layerFinder("mask_fcn_logits/BiasAdd"); + + if (!detectionLayer || !maskLayer) { + std::cerr << "ERROR: some layers missing or unsupported data types " + << "in output tensors" << std::endl; + return false; + } + + if(maskLayer->inferDims.numDims != 4U) { + std::cerr << "Network output number of dims is : " << + maskLayer->inferDims.numDims << " expect is 4"<< std::endl; + return false; + } + + const unsigned int det_max_instances = maskLayer->inferDims.d[0]; + const unsigned int num_classes = maskLayer->inferDims.d[1]; + if(num_classes != detectionParams.numClassesConfigured) { + std::cerr << "WARNING: Num classes mismatch. Configured:" << + detectionParams.numClassesConfigured << ", detected by network: " << + num_classes << std::endl; + } + const unsigned int mask_instance_height= maskLayer->inferDims.d[2]; + const unsigned int mask_instance_width = maskLayer->inferDims.d[3]; + + auto out_det = reinterpret_cast( detectionLayer->buffer); + auto out_mask = reinterpret_cast(maskLayer->buffer); + + for(auto i = 0U; i < det_max_instances; i++) { + MrcnnRawDetection &rawDec = out_det[i]; + + if(rawDec.score < detectionParams.perClassPreclusterThreshold[0]) + continue; + + NvDsInferInstanceMaskInfo obj; + obj.left = CLIP(rawDec.x1, 0, networkInfo.width - 1); + obj.top = CLIP(rawDec.y1, 0, networkInfo.height - 1); + obj.width = CLIP(rawDec.x2, 0, networkInfo.width - 1) - rawDec.x1; + obj.height = CLIP(rawDec.y2, 0, networkInfo.height - 1) - rawDec.y1; + if(obj.width <= 0 || obj.height <= 0) + continue; + obj.classId = static_cast(rawDec.class_id); + obj.detectionConfidence = rawDec.score; + + obj.mask_size = sizeof(float)*mask_instance_width*mask_instance_height; + obj.mask = new float[mask_instance_width*mask_instance_height]; + obj.mask_width = mask_instance_width; + obj.mask_height = mask_instance_height; + + float *rawMask = reinterpret_cast(out_mask + i + * detectionParams.numClassesConfigured + obj.classId); + memcpy (obj.mask, rawMask, sizeof(float)*mask_instance_width*mask_instance_height); + + objectList.push_back(obj); + } + + return true; + +} + +extern "C" +bool NvDsInferParseCustomDDETRTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + + // Code from NvDsInferParseCustomTfSSD for layer finding + auto layerFinder = [&outputLayersInfo](const std::string &name) + -> const NvDsInferLayerInfo *{ + for (auto &layer : outputLayersInfo) { + if (layer.dataType == FLOAT && + (layer.layerName && name == layer.layerName)) { + return &layer; + } + } + return nullptr; + }; + + const NvDsInferLayerInfo *boxLayer = layerFinder("pred_boxes"); // 1 x num_queries x 4 + const NvDsInferLayerInfo *classLayer = layerFinder("pred_logits"); // 1 x num_queries x num_classes + + if (!boxLayer || !classLayer) { + std::cerr << "ERROR: some layers missing or unsupported data types " + << "in output tensors" << std::endl; + return false; + } + + const int keep_top_k = 200; + unsigned int numDetections = classLayer->inferDims.d[0]; + unsigned int numClasses = classLayer->inferDims.d[1]; + std::map ordered_objects; + + for (unsigned int idx = 0; idx < numDetections; idx += 1) { + NvDsInferObjectDetectionInfo res = {}; + + unsigned int class_layer_idx = idx * numClasses; + + res.classId = std::max_element(((float*)classLayer->buffer+class_layer_idx), ((float*)classLayer->buffer+class_layer_idx+numClasses)) - ((float*)classLayer->buffer+class_layer_idx); + res.detectionConfidence = ((float*)classLayer->buffer)[class_layer_idx+res.classId]; + + // If model does not have sigmoid layer, perform sigmoid calculation here + res.detectionConfidence = 1.0/(1.0 + exp(-res.detectionConfidence)); + + if(res.classId == 0 || res.detectionConfidence < detectionParams.perClassPreclusterThreshold[res.classId]) { + continue; + } + enum {cx, cy, w, h}; + float rectX1f, rectY1f, rectX2f, rectY2f; + + unsigned int box_layer_idx = idx * 4; + + rectX1f = (((float*)boxLayer->buffer)[box_layer_idx + cx] - (((float*)boxLayer->buffer)[box_layer_idx + w]/2)) * networkInfo.width; + rectY1f = (((float*)boxLayer->buffer)[box_layer_idx + cy] - (((float*)boxLayer->buffer)[box_layer_idx + h]/2)) * networkInfo.height; + rectX2f = rectX1f + ((float*)boxLayer->buffer)[box_layer_idx + w] * networkInfo.width; + rectY2f = rectY1f + ((float*)boxLayer->buffer)[box_layer_idx + h] * networkInfo.height; + + rectX1f = CLIP(rectX1f, 0.0f, networkInfo.width - 1); + rectX2f = CLIP(rectX2f, 0.0f, networkInfo.width - 1); + rectY1f = CLIP(rectY1f, 0.0f, networkInfo.height - 1); + rectY2f = CLIP(rectY2f, 0.0f, networkInfo.height - 1); + + res.left = rectX1f; + res.top = rectY1f; + res.width = rectX2f - rectX1f; + res.height = rectY2f - rectY1f; + + ordered_objects[res.detectionConfidence] = res; + } + + int jdx = 0; + for (auto iter=ordered_objects.rbegin(); iter!=ordered_objects.rend() && jdxsecond.classId != 0){ + objectList.emplace_back(iter->second);} + } + return true; +} + + +extern "C" +bool NvDsInferParseCustomEfficientDetTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + if(outputLayersInfo.size() != 4) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 4 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + int* p_keep_count = (int *) outputLayersInfo[0].buffer; + + float* p_bboxes = (float *) outputLayersInfo[1].buffer; + NvDsInferDims inferDims_p_bboxes = outputLayersInfo[1].inferDims; + int numElements_p_bboxes=inferDims_p_bboxes.numElements; + + float* p_scores = (float *) outputLayersInfo[2].buffer; + float* p_classes = (float *) outputLayersInfo[3].buffer; + + const int out_class_size = detectionParams.numClassesConfigured; + const float threshold = detectionParams.perClassThreshold[0]; + + float max_bbox=0; + for (int i=0; i < numElements_p_bboxes; i++) + { + // std::cout <<"p_bboxes: " + // < 0) + { + assert (normalized == 0); + for (int i = 0; i < p_keep_count[0]; i++) { + + + if ( p_scores[i] < threshold) continue; + assert((int) p_classes[i] < out_class_size); + + + // std::cout << "label/conf/ x/y x/y -- " + // << (int)p_classes[i] << " " << p_scores[i] << " " + // << p_bboxes[4*i] << " " << p_bboxes[4*i+1] << " " << p_bboxes[4*i+2] << " "<< p_bboxes[4*i+3] << " " << std::endl; + + if(p_bboxes[4*i+2] < p_bboxes[4*i] || p_bboxes[4*i+3] < p_bboxes[4*i+1]) + continue; + + NvDsInferObjectDetectionInfo object = {}; + object.classId = (int) p_classes[i]; + object.detectionConfidence = p_scores[i]; + + + object.left=p_bboxes[4*i+1]; + object.top=p_bboxes[4*i]; + object.width=( p_bboxes[4*i+3] - object.left); + object.height= ( p_bboxes[4*i+2] - object.top); + + object.left=CLIP(object.left, 0, networkInfo.width - 1); + object.top=CLIP(object.top, 0, networkInfo.height - 1); + object.width=CLIP(object.width, 0, networkInfo.width - 1); + object.height=CLIP(object.height, 0, networkInfo.height - 1); + + objectList.push_back(object); + } + } + return true; +} + + +/* Check that the custom function has been defined correctly */ +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomNMSTLT); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomBatchedNMSTLT); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomDDETRTAO); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomEfficientDetTAO); diff --git a/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/detector_labels.txt b/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/detector_labels.txt new file mode 100644 index 0000000..4b02dbb --- /dev/null +++ b/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/detector_labels.txt @@ -0,0 +1,4 @@ +BG +Person +Face +Bag diff --git a/deepstream-tracker-3d-multi-view/models/RTDETR/custom_parser/Makefile b/deepstream-tracker-3d-multi-view/models/RTDETR/custom_parser/Makefile new file mode 100644 index 0000000..18d8e1f --- /dev/null +++ b/deepstream-tracker-3d-multi-view/models/RTDETR/custom_parser/Makefile @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + + +DS_VER = $(shell deepstream-app -v | awk '$$1~/DeepStreamSDK/ {print substr($$2,1,3)}' ) + +DS_SRC_PATH := /opt/nvidia/deepstream/deepstream-$(DS_VER) +CC:= g++ + +# Change to your deepstream SDK includes +CFLAGS+= -I$(DS_SRC_PATH)/sources/includes \ + -I/usr/local/cuda/include + +CFLAGS+= -Wall -std=c++11 -shared -fPIC + +LIBS+= -lnvinfer -L/usr/local/cuda/lib64 -lcudart -lcublas + +LFLAGS:= -Wl,--start-group $(LIBS) -Wl,--end-group + +SRCFILES:= nvdsinfer_custombboxparser_tao.cpp +TARGET_LIB:= libnvds_infercustomparser_tao.so + +all: $(TARGET_LIB) + +$(TARGET_LIB) : $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS) $(LFLAGS) + +clean: + rm -rf $(TARGET_LIB) diff --git a/deepstream-tracker-3d-multi-view/models/RTDETR/custom_parser/nvdsinfer_custombboxparser_tao.cpp b/deepstream-tracker-3d-multi-view/models/RTDETR/custom_parser/nvdsinfer_custombboxparser_tao.cpp new file mode 100644 index 0000000..c89a71c --- /dev/null +++ b/deepstream-tracker-3d-multi-view/models/RTDETR/custom_parser/nvdsinfer_custombboxparser_tao.cpp @@ -0,0 +1,425 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include "nvdsinfer_custom_impl.h" +#include +#include +#include +#include + +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define MAX(a,b) ((a) > (b) ? (a) : (b)) +#define CLIP(a,min,max) (MAX(MIN(a, max), min)) +#define DIVIDE_AND_ROUND_UP(a, b) ((a + b - 1) / b) + +struct MrcnnRawDetection { + float y1, x1, y2, x2, class_id, score; +}; +/* This is a sample bounding box parsing function for the sample FasterRCNN + * + * detector model provided with the SDK. */ + +/* C-linkage to prevent name-mangling */ +extern "C" +bool NvDsInferParseCustomNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomBatchedNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomDDETRTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomEfficientDetTAO ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + + +extern "C" +bool NvDsInferParseCustomNMSTLT (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + if(outputLayersInfo.size() != 2) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 2 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + // Host memory for "nms" which has 2 output bindings: + // the order is bboxes and keep_count + float* out_nms = (float *) outputLayersInfo[0].buffer; + int * p_keep_count = (int *) outputLayersInfo[1].buffer; + const int out_class_size = detectionParams.numClassesConfigured; + const float threshold = detectionParams.perClassThreshold[0]; + + float* det; + + for (int i = 0; i < p_keep_count[0]; i++) { + det = out_nms + i * 7; + + // Output format for each detection is stored in the below order + // [image_id, label, confidence, xmin, ymin, xmax, ymax] + if ( det[2] < threshold) continue; + assert((int) det[1] < out_class_size); + +#if 0 + std::cout << "id/label/conf/ x/y x/y -- " + << det[0] << " " << det[1] << " " << det[2] << " " + << det[3] << " " << det[4] << " " << det[5] << " " << det[6] << std::endl; +#endif + NvDsInferObjectDetectionInfo object = {}; + object.classId = (int) det[1]; + object.detectionConfidence = det[2]; + + /* Clip object box co-ordinates to network resolution */ + object.left = CLIP(det[3] * networkInfo.width, 0, networkInfo.width - 1); + object.top = CLIP(det[4] * networkInfo.height, 0, networkInfo.height - 1); + object.width = CLIP((det[5] - det[3]) * networkInfo.width, 0, networkInfo.width - 1); + object.height = CLIP((det[6] - det[4]) * networkInfo.height, 0, networkInfo.height - 1); + + objectList.push_back(object); + } + + return true; +} + +extern "C" +bool NvDsInferParseCustomBatchedNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + + if(outputLayersInfo.size() != 4) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 4 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + /* Host memory for "BatchedNMS" + BatchedNMS has 4 output bindings, the order is: + keepCount, bboxes, scores, classes + */ + int* p_keep_count = (int *) outputLayersInfo[0].buffer; + float* p_bboxes = (float *) outputLayersInfo[1].buffer; + float* p_scores = (float *) outputLayersInfo[2].buffer; + float* p_classes = (float *) outputLayersInfo[3].buffer; + + const float threshold = detectionParams.perClassThreshold[0]; + + const int keep_top_k = 200; + const char* log_enable = std::getenv("ENABLE_DEBUG"); + + if(log_enable != NULL && std::stoi(log_enable)) { + std::cout <<"keep cout" + <= detectionParams.numClassesConfigured) continue; + if(p_bboxes[4*i+2] < p_bboxes[4*i] || p_bboxes[4*i+3] < p_bboxes[4*i+1]) continue; + + NvDsInferObjectDetectionInfo object = {}; + object.classId = (int) p_classes[i]; + object.detectionConfidence = p_scores[i]; + + /* Clip object box co-ordinates to network resolution */ + object.left = CLIP(p_bboxes[4*i] * networkInfo.width, 0, networkInfo.width - 1); + object.top = CLIP(p_bboxes[4*i+1] * networkInfo.height, 0, networkInfo.height - 1); + object.width = CLIP(p_bboxes[4*i+2] * networkInfo.width, 0, networkInfo.width - 1) - object.left; + object.height = CLIP(p_bboxes[4*i+3] * networkInfo.height, 0, networkInfo.height - 1) - object.top; + + if(object.height < 0 || object.width < 0) + continue; + objectList.push_back(object); + } + return true; +} + +extern "C" +bool NvDsInferParseCustomMrcnnTLTV2 (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + auto layerFinder = [&outputLayersInfo](const std::string &name) + -> const NvDsInferLayerInfo *{ + for (auto &layer : outputLayersInfo) { + if (layer.dataType == FLOAT && + (layer.layerName && name == layer.layerName)) { + return &layer; + } + } + return nullptr; + }; + + const NvDsInferLayerInfo *detectionLayer = layerFinder("generate_detections"); + const NvDsInferLayerInfo *maskLayer = layerFinder("mask_fcn_logits/BiasAdd"); + + if (!detectionLayer || !maskLayer) { + std::cerr << "ERROR: some layers missing or unsupported data types " + << "in output tensors" << std::endl; + return false; + } + + if(maskLayer->inferDims.numDims != 4U) { + std::cerr << "Network output number of dims is : " << + maskLayer->inferDims.numDims << " expect is 4"<< std::endl; + return false; + } + + const unsigned int det_max_instances = maskLayer->inferDims.d[0]; + const unsigned int num_classes = maskLayer->inferDims.d[1]; + if(num_classes != detectionParams.numClassesConfigured) { + std::cerr << "WARNING: Num classes mismatch. Configured:" << + detectionParams.numClassesConfigured << ", detected by network: " << + num_classes << std::endl; + } + const unsigned int mask_instance_height= maskLayer->inferDims.d[2]; + const unsigned int mask_instance_width = maskLayer->inferDims.d[3]; + + auto out_det = reinterpret_cast( detectionLayer->buffer); + auto out_mask = reinterpret_cast(maskLayer->buffer); + + for(auto i = 0U; i < det_max_instances; i++) { + MrcnnRawDetection &rawDec = out_det[i]; + + if(rawDec.score < detectionParams.perClassPreclusterThreshold[0]) + continue; + + NvDsInferInstanceMaskInfo obj; + obj.left = CLIP(rawDec.x1, 0, networkInfo.width - 1); + obj.top = CLIP(rawDec.y1, 0, networkInfo.height - 1); + obj.width = CLIP(rawDec.x2, 0, networkInfo.width - 1) - rawDec.x1; + obj.height = CLIP(rawDec.y2, 0, networkInfo.height - 1) - rawDec.y1; + if(obj.width <= 0 || obj.height <= 0) + continue; + obj.classId = static_cast(rawDec.class_id); + obj.detectionConfidence = rawDec.score; + + obj.mask_size = sizeof(float)*mask_instance_width*mask_instance_height; + obj.mask = new float[mask_instance_width*mask_instance_height]; + obj.mask_width = mask_instance_width; + obj.mask_height = mask_instance_height; + + float *rawMask = reinterpret_cast(out_mask + i + * detectionParams.numClassesConfigured + obj.classId); + memcpy (obj.mask, rawMask, sizeof(float)*mask_instance_width*mask_instance_height); + + objectList.push_back(obj); + } + + return true; + +} + +extern "C" +bool NvDsInferParseCustomDDETRTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + + // Code from NvDsInferParseCustomTfSSD for layer finding + auto layerFinder = [&outputLayersInfo](const std::string &name) + -> const NvDsInferLayerInfo *{ + for (auto &layer : outputLayersInfo) { + if (layer.dataType == FLOAT && + (layer.layerName && name == layer.layerName)) { + return &layer; + } + } + return nullptr; + }; + + const NvDsInferLayerInfo *boxLayer = layerFinder("pred_boxes"); // 1 x num_queries x 4 + const NvDsInferLayerInfo *classLayer = layerFinder("pred_logits"); // 1 x num_queries x num_classes + + if (!boxLayer || !classLayer) { + std::cerr << "ERROR: some layers missing or unsupported data types " + << "in output tensors" << std::endl; + return false; + } + + const int keep_top_k = 200; + unsigned int numDetections = classLayer->inferDims.d[0]; + unsigned int numClasses = classLayer->inferDims.d[1]; + std::map ordered_objects; + + for (unsigned int idx = 0; idx < numDetections; idx += 1) { + NvDsInferObjectDetectionInfo res = {}; + + unsigned int class_layer_idx = idx * numClasses; + + res.classId = std::max_element(((float*)classLayer->buffer+class_layer_idx), ((float*)classLayer->buffer+class_layer_idx+numClasses)) - ((float*)classLayer->buffer+class_layer_idx); + res.detectionConfidence = ((float*)classLayer->buffer)[class_layer_idx+res.classId]; + + // If model does not have sigmoid layer, perform sigmoid calculation here + res.detectionConfidence = 1.0/(1.0 + exp(-res.detectionConfidence)); + + // Skip forklift and pallet, refer detector_labels.txt for the class ids + // if (res.classId != 0 && res.classId != 1 && res.classId != 2 && res.classId != 3 && res.classId != 4 && res.classId != 5) { + // continue; + // } + if(res.detectionConfidence < detectionParams.perClassPreclusterThreshold[res.classId]) { + continue; + } + enum {cx, cy, w, h}; + float rectX1f, rectY1f, rectX2f, rectY2f; + + unsigned int box_layer_idx = idx * 4; + + rectX1f = (((float*)boxLayer->buffer)[box_layer_idx + cx] - (((float*)boxLayer->buffer)[box_layer_idx + w]/2)) * networkInfo.width; + rectY1f = (((float*)boxLayer->buffer)[box_layer_idx + cy] - (((float*)boxLayer->buffer)[box_layer_idx + h]/2)) * networkInfo.height; + rectX2f = rectX1f + ((float*)boxLayer->buffer)[box_layer_idx + w] * networkInfo.width; + rectY2f = rectY1f + ((float*)boxLayer->buffer)[box_layer_idx + h] * networkInfo.height; + + rectX1f = CLIP(rectX1f, 0.0f, networkInfo.width - 1); + rectX2f = CLIP(rectX2f, 0.0f, networkInfo.width - 1); + rectY1f = CLIP(rectY1f, 0.0f, networkInfo.height - 1); + rectY2f = CLIP(rectY2f, 0.0f, networkInfo.height - 1); + + res.left = rectX1f; + res.top = rectY1f; + res.width = rectX2f - rectX1f; + res.height = rectY2f - rectY1f; + + ordered_objects[res.detectionConfidence] = res; + } + + int jdx = 0; + for (auto iter=ordered_objects.rbegin(); iter!=ordered_objects.rend() && jdxsecond); + } + return true; +} + + +extern "C" +bool NvDsInferParseCustomEfficientDetTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + if(outputLayersInfo.size() != 4) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 4 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + int* p_keep_count = (int *) outputLayersInfo[0].buffer; + + float* p_bboxes = (float *) outputLayersInfo[1].buffer; + NvDsInferDims inferDims_p_bboxes = outputLayersInfo[1].inferDims; + int numElements_p_bboxes=inferDims_p_bboxes.numElements; + + float* p_scores = (float *) outputLayersInfo[2].buffer; + float* p_classes = (float *) outputLayersInfo[3].buffer; + + const int out_class_size = detectionParams.numClassesConfigured; + const float threshold = detectionParams.perClassThreshold[0]; + + float max_bbox=0; + for (int i=0; i < numElements_p_bboxes; i++) + { + // std::cout <<"p_bboxes: " + // < 0) + { + assert (normalized == 0); + for (int i = 0; i < p_keep_count[0]; i++) { + + + if ( p_scores[i] < threshold) continue; + assert((int) p_classes[i] < out_class_size); + + + // std::cout << "label/conf/ x/y x/y -- " + // << (int)p_classes[i] << " " << p_scores[i] << " " + // << p_bboxes[4*i] << " " << p_bboxes[4*i+1] << " " << p_bboxes[4*i+2] << " "<< p_bboxes[4*i+3] << " " << std::endl; + + if(p_bboxes[4*i+2] < p_bboxes[4*i] || p_bboxes[4*i+3] < p_bboxes[4*i+1]) + continue; + + NvDsInferObjectDetectionInfo object = {}; + object.classId = (int) p_classes[i]; + object.detectionConfidence = p_scores[i]; + + + object.left=p_bboxes[4*i+1]; + object.top=p_bboxes[4*i]; + object.width=( p_bboxes[4*i+3] - object.left); + object.height= ( p_bboxes[4*i+2] - object.top); + + object.left=CLIP(object.left, 0, networkInfo.width - 1); + object.top=CLIP(object.top, 0, networkInfo.height - 1); + object.width=CLIP(object.width, 0, networkInfo.width - 1); + object.height=CLIP(object.height, 0, networkInfo.height - 1); + + objectList.push_back(object); + } + } + return true; +} + + +/* Check that the custom function has been defined correctly */ +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomNMSTLT); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomBatchedNMSTLT); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomDDETRTAO); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomEfficientDetTAO); diff --git a/deepstream-tracker-3d-multi-view/models/RTDETR/detector_labels.txt b/deepstream-tracker-3d-multi-view/models/RTDETR/detector_labels.txt new file mode 100644 index 0000000..b10e95b --- /dev/null +++ b/deepstream-tracker-3d-multi-view/models/RTDETR/detector_labels.txt @@ -0,0 +1,7 @@ +person +agility_digit +gr1_t2 +nova_carter +transporter +forklift +pallet diff --git a/deepstream-tracker-3d-multi-view/requirements.txt b/deepstream-tracker-3d-multi-view/requirements.txt new file mode 100644 index 0000000..a4aeccc --- /dev/null +++ b/deepstream-tracker-3d-multi-view/requirements.txt @@ -0,0 +1,6 @@ +kafka-python>=2.0.2 +protobuf>=3.20.0 +numpy>=1.21.0 +PyYAML>=6.0 +opencv-python>=4.5.0 +tqdm>=4.64.0 \ No newline at end of file diff --git a/deepstream-tracker-3d-multi-view/scripts/check_prerequisites.sh b/deepstream-tracker-3d-multi-view/scripts/check_prerequisites.sh new file mode 100755 index 0000000..725f56a --- /dev/null +++ b/deepstream-tracker-3d-multi-view/scripts/check_prerequisites.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +# Global variables - same as setup script +BASE_DIR=${BASE_DIR:-$HOME} +USE_INFERENCE_BUILDER=${USE_INFERENCE_BUILDER:-false} +KAFKA_VERSION="4.2.0" +SCALA_VERSION="2.13" + +# Standardized paths +KAFKA_DIR="$BASE_DIR/kafka_${SCALA_VERSION}-${KAFKA_VERSION}" +INFERENCE_BUILDER_DIR="$BASE_DIR/inference_builder" + +# Initialize status variables +GPU_STATUS="✓" +MQTT_STATUS="✓" +KAFKA_STATUS="✓" +MV3DT_VENV_STATUS="✓" +INFERENCE_BUILDER_STATUS="✓" +DATASETS_MODELS_STATUS="✓" + +# Check 1: Check if NVIDIA GPU is available +if ! nvidia-smi > /dev/null 2>&1; then + echo "NVIDIA GPU is not available" + GPU_STATUS="✗" +fi + +# Check 2: Check if MQTT broker is running on port 1883 +if ! ./scripts/mosquitto_test.sh > /dev/null 2>&1; then + MQTT_STATUS="✗" +fi + +# Check 3: Check if Kafka is running +if [ ! -d "$KAFKA_DIR" ]; then + echo "Kafka directory not found: $KAFKA_DIR" + KAFKA_STATUS="✗" +else + KAFKA_TOPICS_SCRIPT="$KAFKA_DIR/bin/kafka-topics.sh" + if [ -x "$KAFKA_TOPICS_SCRIPT" ]; then + TOPIC_LIST=$(timeout 5s $KAFKA_TOPICS_SCRIPT --bootstrap-server localhost:9092 --list 2>/dev/null) + if ! echo "$TOPIC_LIST" | grep -q "mv3dt"; then + echo "Kafka topics: $TOPIC_LIST" + echo "mv3dt topic not found" + KAFKA_STATUS="✗" + fi + else + echo "Kafka topics script not found or not executable: $KAFKA_TOPICS_SCRIPT" + KAFKA_STATUS="✗" + fi +fi + +# Check 4: Check whether virtual env "mv3dt_venv" setup correctly +if [ ! -d "$PWD/mv3dt_venv" ]; then + echo "Virtual environment 'mv3dt_venv' not found" + MV3DT_VENV_STATUS="✗" +fi + +# Check 5: Check whether inference builder is setup correctly +if [[ "$USE_INFERENCE_BUILDER" != "true" ]]; then + INFERENCE_BUILDER_STATUS="(skipped)" +else + if [ ! -d "$INFERENCE_BUILDER_DIR" ]; then + echo "Inference builder directory not found: $INFERENCE_BUILDER_DIR" + INFERENCE_BUILDER_STATUS="✗" + elif [ ! -d "$INFERENCE_BUILDER_DIR/ib_venv" ]; then + echo "Inference builder virtual environment 'ib_venv' not found: $INFERENCE_BUILDER_DIR/ib_venv" + INFERENCE_BUILDER_STATUS="✗" + fi +fi + +# Check 6: check wether datasets and models are setup correctly +if [ ! -d "$PWD/datasets" ]; then + echo "Datasets directory not found" + DATASETS_MODELS_STATUS="✗" +else + if [ ! -d "$PWD/datasets/mtmc_4cam/camInfo" ]; then + echo "datasets/mtmc_4cam/camInfo directory not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -d "$PWD/datasets/mtmc_4cam/videos" ]; then + echo "datasets/mtmc_4cam/videos directory not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -d "$PWD/datasets/mtmc_12cam/camInfo" ]; then + echo "datasets/mtmc_12cam/camInfo directory not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -d "$PWD/datasets/mtmc_12cam/videos" ]; then + echo "datasets/mtmc_12cam/videos directory not found" + DATASETS_MODELS_STATUS="✗" + fi +fi + +if [ ! -d "$PWD/models" ]; then + echo "Models directory not found" + DATASETS_MODELS_STATUS="✗" +else + if [ ! -d "$PWD/models/BodyPose3DNet" ]; then + echo "BodyPose3DNet model not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -d "$PWD/models/PeopleNetTransformer" ]; then + echo "PeopleNetTransformer model not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -f "$PWD/models/PeopleNetTransformer/custom_parser/libnvds_infercustomparser_tao.so" ]; then + echo "PeopleNetTransformer custom parser libnvds_infercustomparser_tao.so not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -d "$PWD/models/RTDETR" ]; then + echo "RTDETR model not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -f "$PWD/models/RTDETR/custom_parser/libnvds_infercustomparser_tao.so" ]; then + echo "RTDETR custom parser libnvds_infercustomparser_tao.so not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -d "$PWD/models/PeopleNet2.6.3" ]; then + echo "PeopleNet2.6.3 model not found" + DATASETS_MODELS_STATUS="✗" + fi +fi + +# Summary of all checks +echo "" +echo "---- PREREQUISITES CHECK SUMMARY ----" +echo "1. NVIDIA GPU: $GPU_STATUS" +echo "2. MQTT Broker: $MQTT_STATUS" +echo "3. Kafka Broker: $KAFKA_STATUS" +echo "4. Python Environment: $MV3DT_VENV_STATUS" +echo "5. Datasets & Models: $DATASETS_MODELS_STATUS" +if [[ "$USE_INFERENCE_BUILDER" == "true" ]]; then + echo "6. Inference Builder: $INFERENCE_BUILDER_STATUS" +fi +echo "-------------------------------------" + +# Check overall status and exit with appropriate code +inference_check_passed=true +if [[ "$USE_INFERENCE_BUILDER" == "true" && "$INFERENCE_BUILDER_STATUS" != "✓" ]]; then + inference_check_passed=false +fi + +if [[ "$GPU_STATUS" = "✓" && "$MQTT_STATUS" = "✓" && "$KAFKA_STATUS" = "✓" && + "$MV3DT_VENV_STATUS" = "✓" && "$DATASETS_MODELS_STATUS" = "✓" && + "$inference_check_passed" = true ]]; then + echo "" + echo "✅ All prerequisites are properly set up!" + exit 0 +else + echo "" + echo "❌ Some prerequisites are missing or not properly configured." + echo "Please run the setup script or follow manual setup instructions." + exit 1 +fi \ No newline at end of file diff --git a/deepstream-tracker-3d-multi-view/scripts/create_6cam_subset.sh b/deepstream-tracker-3d-multi-view/scripts/create_6cam_subset.sh new file mode 100755 index 0000000..701b9e3 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/scripts/create_6cam_subset.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Simple script to create a random 6-camera subset from 12-camera dataset + +set -e + +DATASET_DIR="datasets" +SOURCE_DIR="$DATASET_DIR/mtmc_12cam" +TARGET_DIR="$DATASET_DIR/mtmc_6cam" + +# Check if source dataset exists +if [[ ! -d "$SOURCE_DIR" ]]; then + echo "Error: Source dataset not found: $SOURCE_DIR" + exit 1 +fi + +# Create target directory +if [[ -d "$TARGET_DIR" ]]; then + echo "Target directory already exists: $TARGET_DIR" + read -p "Remove existing directory and continue? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + rm -rf "$TARGET_DIR" + else + exit 1 + fi +fi + +echo "Creating 6-camera subset from 12-camera dataset..." + +# Create directory structure +mkdir -p "$TARGET_DIR/camInfo" +mkdir -p "$TARGET_DIR/videos" + +# Randomly select 6 cameras from 1-12 +selected_cams=($(seq -f "%03g" 1 12 | shuf -n 6 | sort)) + +echo "Selected cameras: ${selected_cams[@]}" + +# Copy selected camera files +for cam in "${selected_cams[@]}"; do + cam_file="Warehouse_Synthetic_Cam${cam}" + + # Copy camera info file + if [[ -f "$SOURCE_DIR/camInfo/${cam_file}.yml" ]]; then + cp "$SOURCE_DIR/camInfo/${cam_file}.yml" "$TARGET_DIR/camInfo/" + echo "Copied: camInfo/${cam_file}.yml" + fi + + # Copy video file + if [[ -f "$SOURCE_DIR/videos/${cam_file}.mp4" ]]; then + cp "$SOURCE_DIR/videos/${cam_file}.mp4" "$TARGET_DIR/videos/" + echo "Copied: videos/${cam_file}.mp4" + fi +done + +# Copy common files +cp "$SOURCE_DIR/map.png" "$TARGET_DIR/" +cp "$SOURCE_DIR/transforms.yml" "$TARGET_DIR/" + +echo "" +echo "✅ Successfully created 6-camera dataset at: $TARGET_DIR" +echo "📁 Selected cameras: ${selected_cams[@]}" +echo "" +echo "To use this subset, update your experiment config to point to: $TARGET_DIR" diff --git a/deepstream-tracker-3d-multi-view/scripts/mosquitto_test.sh b/deepstream-tracker-3d-multi-view/scripts/mosquitto_test.sh new file mode 100755 index 0000000..0a496c4 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/scripts/mosquitto_test.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Test parameters +BROKER="localhost" # Change if the broker is on a different machine or IP address +PORT=1883 # Default MQTT port for Mosquitto +TOPIC="test/topic" # The topic to test +MESSAGE="Hello from Mosquitto test!" # The test message + + +# Subscribe to the topic in the background +echo "Subscribing to topic '$TOPIC'..." +mosquitto_sub -h $BROKER -p $PORT -t $TOPIC & +SUB_PID=$! + +# Give the subscriber some time to start up +sleep 1 + +# Publish a message to the topic +echo "Publishing message to topic '$TOPIC'..." +mosquitto_pub -h $BROKER -p $PORT -t $TOPIC -m "$MESSAGE" + +# Wait a few seconds to allow the subscriber to receive the message +sleep 2 + +# Stop the mosquitto_sub process +kill $SUB_PID +wait $SUB_PID + +# The script will automatically exit, and the message should be received by the subscriber. +echo "Test completed." diff --git a/deepstream-tracker-3d-multi-view/scripts/setup_prerequisites.sh b/deepstream-tracker-3d-multi-view/scripts/setup_prerequisites.sh new file mode 100755 index 0000000..1027e2e --- /dev/null +++ b/deepstream-tracker-3d-multi-view/scripts/setup_prerequisites.sh @@ -0,0 +1,724 @@ +#!/bin/bash + +# setup_prerequisites.sh - Automated prerequisites setup for MV3DT +# This script automates all the manual setup steps described in README.md + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } +log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; } +log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } + +# Global variables +export DEEPSTREAM_IMAGE="${DEEPSTREAM_IMAGE:-nvcr.io/nvidia/deepstream:9.0-triton-multiarch}" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BASE_DIR=${BASE_DIR:-$HOME} +USE_INFERENCE_BUILDER=${USE_INFERENCE_BUILDER:-false} +KAFKA_VERSION="4.2.0" +SCALA_VERSION="2.13" + +# Standardized paths +KAFKA_DIR="$BASE_DIR/kafka_${SCALA_VERSION}-${KAFKA_VERSION}" +INFERENCE_BUILDER_DIR="$BASE_DIR/inference_builder" + +# Utility functions +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +check_os() { + if [[ "$OSTYPE" != "linux-gnu"* ]]; then + log_error "This script only supports Linux. Detected OS: $OSTYPE" + exit 1 + fi + + if ! command_exists lsb_release; then + log_warning "lsb_release not found, checking if lsb-release package is installed" + if ! dpkg-query -W -f='${Status}' lsb-release 2>/dev/null | grep -q "install ok installed"; then + log_info "Installing lsb-release package..." + sudo apt update && sudo apt install -y lsb-release || true + else + log_info "lsb-release package is installed but command not found, may need PATH update" + fi + fi + + if command_exists lsb_release; then + local distro=$(lsb_release -si) + local version=$(lsb_release -sr) + log_info "Detected OS: $distro $version" + + if [[ "$distro" != "Ubuntu" ]]; then + log_warning "This script is optimized for Ubuntu 24.04, but will attempt to continue on $distro" + fi + fi +} + +check_gpu() { + log_info "Checking NVIDIA GPU availability..." + if command_exists nvidia-smi; then + if nvidia-smi >/dev/null 2>&1; then + local driver_version=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader,nounits | head -1) + local major_version=$(echo "$driver_version" | cut -d'.' -f1) + + if [[ $major_version -ge 580 ]]; then + log_success "NVIDIA GPU detected with driver version: $driver_version" + return 0 + else + log_error "NVIDIA driver version $driver_version is too old (need 580+ series)" + return 1 + fi + else + log_error "nvidia-smi command failed to execute" + fi + else + log_error "nvidia-smi not found" + fi + + log_error "NVIDIA GPU or drivers not properly installed" + log_info "Please install NVIDIA drivers (version 580.xx or 590.xx) and try again" + log_info "Visit: https://docs.nvidia.com/cuda/cuda-installation-guide-linux/" + return 1 +} + + +setup_deepstream_container() { + # Check if DeepStream container is already available + if docker images --format "table {{.Repository}}:{{.Tag}}" | grep -q "^$DEEPSTREAM_IMAGE$"; then + log_success "DeepStream container already available" + else + log_info "Pulling DeepStream container image..." + if docker pull $DEEPSTREAM_IMAGE; then + log_success "DeepStream container pulled successfully" + else + log_error "Failed to pull DeepStream container" + log_info "Please check your network connection and Docker credentials" + return 1 + fi + fi +} + +setup_git_lfs() { + log_info "Setting up Git LFS and extracting datasets and models..." + + cd "$REPO_ROOT" + + # Check if datasets are already extracted + if [[ -d "datasets/mtmc_4cam" && -d "datasets/mtmc_12cam" ]]; then + log_success "Datasets already extracted" + else + # Extract assets if they exist + if [[ -f "assets/datasets.zip" ]]; then + log_info "Extracting datasets..." + unzip -q assets/datasets.zip + log_success "Datasets extracted" + else + log_error "assets/datasets.zip not found" + return 1 + fi + fi + + if [[ -f "models/PeopleNetTransformer/peoplenet_transformer_model_op17.onnx" && -f "models/BodyPose3DNet/bodypose3dnet_accuracy.onnx" && -f "models/RTDETR/rtdetr_warehouse_v1.0.fp16.onnx" && -f "models/PeopleNet2.6.3/resnet34_peoplenet.onnx" ]]; then + log_success "Models already extracted" + else + log_info "Downloading PeopleNet Transformer model..." + mkdir -p models/PeopleNetTransformer + if ! wget -q --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet_transformer_v2/deployable_v1.0/files?redirect=true&path=dino_fan_small_astro_delta.onnx' -O 'models/PeopleNetTransformer/peoplenet_transformer_model_op17.onnx'; then + log_error "Failed to download PeopleNet Transformer model" + return 1 + fi + + log_info "Downloading RT-DETR model..." + mkdir -p models/RTDETR + if ! wget -q --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/rtdetr_2d_warehouse/deployable_efficientvit_l2_v1.0/files?redirect=true&path=rtdetr_warehouse_v1.0.fp16.onnx' -O 'models/RTDETR/rtdetr_warehouse_v1.0.fp16.onnx'; then + log_error "Failed to download RT-DETR model" + return 1 + fi + + log_info "Downloading PeopleNet v2.6.3 model..." + mkdir -p models/PeopleNet2.6.3 + if ! wget -q --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/deployable_quantized_onnx_v2.6.3/files?redirect=true&path=resnet34_peoplenet.onnx' -O 'models/PeopleNet2.6.3/resnet34_peoplenet.onnx'; then + log_error "Failed to download PeopleNet v2.6.3 model" + return 1 + fi + + log_info "Downloading BodyPose3DNet model..." + mkdir -p models/BodyPose3DNet + if ! wget -q --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/bodypose3dnet/deployable_accuracy_onnx_1.0/files?redirect=true&path=bodypose3dnet_accuracy.onnx' -O 'models/BodyPose3DNet/bodypose3dnet_accuracy.onnx'; then + log_error "Failed to download BodyPose3DNet model" + return 1 + fi + fi +} + +setup_mosquitto() { + log_info "Setting up Mosquitto MQTT broker..." + + if ! command_exists mosquitto; then + log_info "Checking Mosquitto packages..." + local mosquitto_packages=("mosquitto" "mosquitto-clients") + local packages_to_install=() + + for package in "${mosquitto_packages[@]}"; do + if ! dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then + packages_to_install+=("$package") + else + log_info "$package is already installed" + fi + done + + if [[ ${#packages_to_install[@]} -gt 0 ]]; then + log_info "Installing missing Mosquitto packages: ${packages_to_install[*]}" + sudo apt-add-repository -y ppa:mosquitto-dev/mosquitto-ppa + sudo apt update + sudo apt install -y "${packages_to_install[@]}" + else + log_success "All Mosquitto packages are already installed" + fi + fi + + # Configure mosquitto + local mosquitto_conf="/etc/mosquitto/conf.d/mv3dt.conf" + if [[ ! -f "$mosquitto_conf" ]] || ! grep -q "set_tcp_nodelay true" "$mosquitto_conf"; then + log_info "Adding set_tcp_nodelay config to Mosquitto..." + echo "set_tcp_nodelay true" | sudo tee "$mosquitto_conf" > /dev/null + fi + + # Start mosquitto service + if ! systemctl is-active --quiet mosquitto; then + log_info "Starting Mosquitto service..." + sudo systemctl enable mosquitto + sudo systemctl start mosquitto + fi + + # Test MQTT broker + sleep 2 + cd "$REPO_ROOT" + if [[ -x "scripts/mosquitto_test.sh" ]]; then + if ./scripts/mosquitto_test.sh >/dev/null 2>&1; then + log_success "Mosquitto broker is running and accessible" + else + log_warning "Mosquitto service test failed, trying manual start..." + # Try starting manually + mosquitto -p 1883 -d + sleep 2 + if ./scripts/mosquitto_test.sh >/dev/null 2>&1; then + log_success "Mosquitto broker started manually" + else + log_error "Failed to start Mosquitto broker" + return 1 + fi + fi + else + log_warning "Mosquitto test script not found, assuming broker is working" + fi +} + +setup_java() { + log_info "Setting up Java for Kafka..." + + if command_exists java; then + local java_version=$(java -version 2>&1 | head -n 1 | cut -d'"' -f2 | cut -d'.' -f1) + if [[ "$java_version" -ge 17 ]]; then + log_success "Java $java_version already installed" + return 0 + fi + fi + + log_info "Checking OpenJDK 17 package..." + if ! dpkg-query -W -f='${Status}' openjdk-17-jdk 2>/dev/null | grep -q "install ok installed"; then + log_info "Installing OpenJDK 17..." + sudo apt update && sudo apt install -y openjdk-17-jdk + else + log_success "OpenJDK 17 package is already installed" + fi + + # Verify installation + if java -version >/dev/null 2>&1; then + log_success "Java installed successfully" + else + log_error "Java installation failed" + return 1 + fi +} + +setup_kafka() { + log_info "Setting up Kafka..." + + # Check if Kafka is already running + if netstat -ln 2>/dev/null | grep -q ":9092 "; then + log_info "Kafka appears to be running on port 9092, checking topic..." + if check_kafka_topic; then + log_success "Kafka is already running with mv3dt topic" + return 0 + else + log_warning "Kafka is running but mv3dt topic not found, stopping old Kafka broker..." + for old_kafka in "$BASE_DIR"/kafka_*/bin/kafka-server-stop.sh; do + [[ -x "$old_kafka" ]] && "$old_kafka" 2>/dev/null || true + done + sleep 3 + fi + fi + + setup_java + + # Download and extract Kafka if not exists + if [[ ! -d "$KAFKA_DIR" ]]; then + log_info "Downloading Kafka ${KAFKA_VERSION}..." + cd "$BASE_DIR" + + local kafka_url="https://dlcdn.apache.org/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" + + if ! wget -q "$kafka_url" -O "kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz"; then + log_error "Failed to download Kafka from $kafka_url" + return 1 + fi + + log_info "Extracting Kafka..." + tar -xzf "kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" + rm "kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" + log_success "Kafka extracted to $KAFKA_DIR" + fi + + cd "$KAFKA_DIR" + + # Check for existing cluster ID or create new one + local meta_properties="/tmp/kraft-combined-logs/meta.properties" + if [[ -f "$meta_properties" ]]; then + # Extract existing cluster ID from meta.properties + export KAFKA_CLUSTER_ID=$(grep "cluster.id=" "$meta_properties" | cut -d'=' -f2) + if [[ -n "$KAFKA_CLUSTER_ID" ]]; then + log_info "Reusing existing Kafka cluster ID: $KAFKA_CLUSTER_ID" + else + log_warning "Found meta.properties but no cluster.id, generating new one" + export KAFKA_CLUSTER_ID="$(bin/kafka-storage.sh random-uuid)" + fi + else + log_info "No existing Kafka cluster found, generating new cluster ID" + export KAFKA_CLUSTER_ID="$(bin/kafka-storage.sh random-uuid)" + fi + + # Format storage only if meta.properties doesn't exist or is invalid + if [[ ! -f "$meta_properties" || -z "$KAFKA_CLUSTER_ID" ]]; then + log_info "Formatting Kafka storage with cluster ID: $KAFKA_CLUSTER_ID" + if ! bin/kafka-storage.sh format --standalone -t $KAFKA_CLUSTER_ID -c config/server.properties >/dev/null 2>&1; then + log_error "Failed to format Kafka storage" + return 1 + fi + else + log_info "Using existing Kafka storage (skipping format)" + fi + + # Start Kafka + log_info "Starting Kafka..." + + # Start Kafka in background + nohup bin/kafka-server-start.sh config/server.properties >/dev/null 2>&1 & + local kafka_pid=$! + + # Wait for Kafka to start + log_info "Waiting for Kafka to start..." + for i in {1..30}; do + if netstat -ln 2>/dev/null | grep -q ":9092 "; then + log_success "Kafka started successfully" + break + fi + sleep 2 + if ! kill -0 $kafka_pid 2>/dev/null; then + log_error "Kafka startup failed" + return 1 + fi + done + + if ! netstat -ln 2>/dev/null | grep -q ":9092 "; then + log_error "Kafka failed to start on port 9092" + return 1 + fi + + # Create mv3dt topic + create_kafka_topic "$KAFKA_DIR" +} + +check_kafka_topic() { + if [[ ! -d "$KAFKA_DIR" || ! -x "$KAFKA_DIR/bin/kafka-topics.sh" ]]; then + return 1 + fi + + local topics=$("$KAFKA_DIR/bin/kafka-topics.sh" --bootstrap-server localhost:9092 --list 2>/dev/null) + echo "$topics" | grep -q "mv3dt" +} + +create_kafka_topic() { + local kafka_dir="$1" + + log_info "Creating mv3dt topic..." + if "$kafka_dir/bin/kafka-topics.sh" --bootstrap-server localhost:9092 \ + --create --topic mv3dt --partitions 1 --replication-factor 1 \ + --config retention.ms=30000 --if-not-exists >/dev/null 2>&1; then + log_success "mv3dt topic created successfully" + else + log_error "Failed to create mv3dt topic" + return 1 + fi +} + +setup_python_env() { + log_info "Setting up Python environment..." + + cd "$REPO_ROOT" + + # Check and install system packages + log_info "Checking system Python packages..." + local packages_to_install=() + local required_packages=("python3-tk" "python3.12-venv" "python3.12-dev" "python3-pip") + + for package in "${required_packages[@]}"; do + if ! dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then + packages_to_install+=("$package") + else + log_info "$package is already installed" + fi + done + + if [[ ${#packages_to_install[@]} -gt 0 ]]; then + log_info "Installing missing packages: ${packages_to_install[*]}" + sudo apt update + sudo apt install -y "${packages_to_install[@]}" + else + log_success "All required Python packages are already installed" + fi + + # Create virtual environment if it doesn't exist + if [[ ! -d "mv3dt_venv" ]]; then + log_info "Creating mv3dt_venv virtual environment..." + python3 -m venv mv3dt_venv + fi + + # Activate and install requirements + log_info "Installing Python dependencies..." + source mv3dt_venv/bin/activate + + if [[ -f "requirements.txt" ]]; then + pip install --quiet --upgrade pip + pip install --quiet -r requirements.txt + log_success "Python dependencies installed" + else + log_error "requirements.txt not found" + return 1 + fi + + # Verify key packages + python -c "import kafka, google.protobuf" 2>/dev/null + if [[ $? -eq 0 ]]; then + log_success "Python environment setup completed successfully" + else + log_error "Failed to import required Python packages" + return 1 + fi +} + +build_custom_parser() { + log_info "Building custom parsers..." + + # Set correct GPU flag considering diffent platforms + if docker info | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" + elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" + else + echo "No GPU support found in Docker." + exit 1 + fi + + local models=("PeopleNetTransformer" "RTDETR") + for model in "${models[@]}"; do + if [[ -f "$PWD/models/$model/custom_parser/libnvds_infercustomparser_tao.so" ]]; then + log_success "$model custom parser already built" + continue + fi + + log_info "Building custom parser for $model..." + build_output=$(docker run --privileged --rm --net=host $GPU_FLAG \ + -v $PWD/models:/workspace/models \ + -w /workspace/models/$model \ + --entrypoint /bin/bash \ + $DEEPSTREAM_IMAGE \ + -c "cd custom_parser && make clean 2>&1 && make 2>&1" 2>&1) + build_exit_code=$? + + if [[ $build_exit_code -ne 0 ]]; then + log_error "Build failed for $model with errors:" + echo "$build_output" + return 1 + fi + + if [[ -f "$PWD/models/$model/custom_parser/libnvds_infercustomparser_tao.so" ]]; then + log_success "$model custom parser built successfully" + else + log_error "Failed to build $model custom parser: libnvds_infercustomparser_tao.so not found" + echo "$build_output" + return 1 + fi + done +} + + +setup_inference_builder() { + if [[ "$USE_INFERENCE_BUILDER" != "true" ]]; then + log_info "Skipping Inference Builder setup (USE_INFERENCE_BUILDER not set to true)" + return 0 + fi + + log_info "Setting up DeepStream Inference Builder..." + + if [[ ! -d "$INFERENCE_BUILDER_DIR" ]]; then + log_info "Cloning Inference Builder repository..." + cd "$BASE_DIR" + if ! git clone https://github.com/NVIDIA-AI-IOT/inference_builder.git; then + log_warning "Failed to clone Inference Builder repository" + log_info "This is optional - Inference Builder is skipped by default. Set USE_INFERENCE_BUILDER=true to enable it." + return 0 + fi + + cd inference_builder + git submodule update --init --recursive + else + log_success "Inference Builder directory already exists" + cd "$INFERENCE_BUILDER_DIR" + fi + + # Install system dependencies + log_info "Checking protobuf-compiler package..." + if ! dpkg-query -W -f='${Status}' protobuf-compiler 2>/dev/null | grep -q "install ok installed"; then + log_info "Installing protobuf-compiler..." + sudo apt update && sudo apt install -y protobuf-compiler + else + log_success "protobuf-compiler is already installed" + fi + + # Create virtual environment + if [[ ! -d "ib_venv" ]]; then + log_info "Creating Inference Builder virtual environment..." + python -m venv ib_venv + fi + + # Install requirements + log_info "Installing Inference Builder dependencies..." + source ib_venv/bin/activate + + if [[ -f "requirements.txt" ]]; then + pip install --quiet --upgrade pip + pip install --quiet -r requirements.txt + else + log_warning "Inference Builder requirements.txt not found, skipping pip install" + fi + + # Create custom DeepStream Docker image with MV3DT packages (always) + log_info "Installing inference builder dependencies into DeepStream container..." + + # Create Dockerfile in temporary location + cat > ./Dockerfile.mv3dt << EOF +FROM ${DEEPSTREAM_IMAGE} + +# Install IB python packages +RUN pip3 install torch==2.7.0 omegaconf==2.3.0 + +# Set environment variables +ENV GST_PLUGIN_PATH=/opt/nvidia/deepstream/deepstream/lib/gst-plugins +ENV LD_LIBRARY_PATH=/opt/nvidia/deepstream/deepstream/lib:$LD_LIBRARY_PATH +ENV NVSTREAMMUX_ADAPTIVE_BATCHING=yes + +# Set working directory +WORKDIR /mv3dt_app +EOF + + # Build the custom image + # Before building, check whether the image already exists + if docker inspect inference-builder-mv3dt:latest >/dev/null 2>&1; then + log_info "Target docker image inference-builder-mv3dt:latest already exists" + log_success "Inference Builder setup completed" + return 0 + fi + + # Build the Docker image + if docker build -f ./Dockerfile.mv3dt -t inference-builder-mv3dt:latest .; then + log_info "DeepStream container with Inference Builder dependencies saved as docker image inference-builder-mv3dt:latest" + rm -f ./Dockerfile.mv3dt + else + log_error "Failed to build custom DeepStream Docker image" + rm -f ./Dockerfile.mv3dt + return 1 + fi + + log_success "Inference Builder setup completed" +} + +run_prerequisites_check() { + log_info "Running prerequisites check..." + cd "$REPO_ROOT" + + if [[ -x "scripts/check_prerequisites.sh" ]]; then + # Make mosquitto test script executable if it exists + [[ -f "scripts/mosquitto_test.sh" ]] && chmod +x scripts/mosquitto_test.sh + + if bash scripts/check_prerequisites.sh; then + return 0 + else + log_error "Prerequisites check failed" + return 1 + fi + else + log_error "Prerequisites check script not found" + return 1 + fi +} + +print_usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Automated setup script for MV3DT prerequisites. + +OPTIONS: + -h, --help Show this help message + --check-only Only run prerequisites check without setup + +ENVIRONMENT VARIABLES: + BASE_DIR Base directory for installations (default: $HOME) + USE_INFERENCE_BUILDER Set to 'true' to enable Inference Builder setup (default: false) + +EOF +} + +main() { + local check_only=false + + # Parse arguments + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_usage + exit 0 + ;; + --check-only) + check_only=true + shift + ;; + *) + log_error "Unknown option: $1" + print_usage + exit 1 + ;; + esac + done + + echo "==============================================" + echo " MV3DT Prerequisites Setup Script" + echo "==============================================" + echo + + if [[ "$check_only" == "true" ]]; then + run_prerequisites_check + exit $? + fi + + check_os + + # Check if we're in the right directory + if [[ ! -f "$REPO_ROOT/README.md" ]] || ! grep -q "Multi-View 3D Tracking" "$REPO_ROOT/README.md"; then + log_error "Please run this script from the deepstream-tracker-3d-multi-view repository root" + exit 1 + fi + + log_info "Starting automated prerequisites setup..." + + # Run setup steps + local failed_steps=() + + if ! check_gpu; then + failed_steps+=("GPU/NVIDIA drivers") + fi + + if ! setup_deepstream_container; then + failed_steps+=("DeepStream container") + fi + + if ! setup_git_lfs; then + failed_steps+=("Git LFS/Datasets/Models") + fi + + if ! build_custom_parser; then + failed_steps+=("Build Custom parser") + fi + + if ! setup_mosquitto; then + failed_steps+=("Mosquitto MQTT") + fi + + if ! setup_kafka; then + failed_steps+=("Kafka") + fi + + if ! setup_python_env; then + failed_steps+=("Python environment") + fi + + if ! setup_inference_builder; then + failed_steps+=("Inference Builder") + fi + + echo + echo "==============================================" + echo " Setup Summary" + echo "==============================================" + + if [[ ${#failed_steps[@]} -eq 0 ]]; then + log_success "All setup steps completed successfully!" + echo + log_info "Configured Paths:" + echo -e " ${BLUE}Base Directory:${NC} $BASE_DIR" + echo -e " ${BLUE}Kafka Installation:${NC} $KAFKA_DIR" + if [[ "$USE_INFERENCE_BUILDER" == "true" ]]; then + echo -e " ${BLUE}Inference Builder:${NC} $INFERENCE_BUILDER_DIR" + fi + echo -e " ${BLUE}MV3DT Repo:${NC} $REPO_ROOT" + echo + + # Run final check + if run_prerequisites_check; then + log_success "Prerequisites check passed! You're ready to use MV3DT." + echo + else + log_error "Setup completed but prerequisites check still failed" + echo "Please review the error messages and run the script again" + exit 1 + fi + else + log_error "The following setup steps failed:" + for step in "${failed_steps[@]}"; do + echo " - $step" + done + echo + echo "Please review the error messages above and:" + echo " 1. Fix the issues manually" + echo " 2. Run this script again" + echo " 3. Or run specific setup steps as needed" + exit 1 + fi + + echo "==============================================" +} + +# Handle script interruption +trap 'log_error "Setup interrupted by user"; exit 130' INT TERM + +# Run main function +main "$@" \ No newline at end of file diff --git a/deepstream-tracker-3d-multi-view/scripts/test_12cam_ds.sh b/deepstream-tracker-3d-multi-view/scripts/test_12cam_ds.sh new file mode 100755 index 0000000..f6afadd --- /dev/null +++ b/deepstream-tracker-3d-multi-view/scripts/test_12cam_ds.sh @@ -0,0 +1,63 @@ +# Set dataset, model, and experiment directories +export DATASET_DIR=$PWD/datasets/mtmc_12cam/ +export EXPERIMENT_DIR=$PWD/experiments/deepstream/12cam +export MODEL_REPO=$PWD/models + +# Select detector model: PeopleNetTransformer (default), RTDETR, or PeopleNet2.6.3 +export DETECTOR_MODEL=${DETECTOR_MODEL:-PeopleNetTransformer} + +if [ "$DETECTOR_MODEL" = "RTDETR" ]; then + DETECTOR_CONFIG="config_pgie_rt_detr.txt" + TRACKER_CONFIG="config_tracker_tuned_12cam_rt_detr.yml" +elif [ "$DETECTOR_MODEL" = "PeopleNet2.6.3" ]; then + DETECTOR_CONFIG="config_pgie_peoplenet.txt" + TRACKER_CONFIG="config_tracker.yml" +else + DETECTOR_CONFIG="config_pgie.txt" + TRACKER_CONFIG="config_tracker_tuned_12cam.yml" +fi + +echo "Using detector model: $DETECTOR_MODEL (detector=$DETECTOR_CONFIG, tracker=$TRACKER_CONFIG)" + +# Set correct GPU flag considering diffent platforms +if docker info | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" +elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" +else + echo "No GPU support found in Docker." + exit 1 +fi + +# Create directories for output +mkdir -p $EXPERIMENT_DIR/infer-kitti-dump +mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump +mkdir -p $EXPERIMENT_DIR/outVideos + +# Auto-generate DeepStream configuration files +source mv3dt_venv/bin/activate +python utils/deepstream_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --enable-osd \ + --tracker-config=$TRACKER_CONFIG \ + --enable-msg-broker \ + --detector-config=$DETECTOR_CONFIG \ + --output-dir=$EXPERIMENT_DIR + +# Launch real-time BEV visualization +python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids & + +# Launch MV3DT pipeline +docker run -t --privileged --rm --net=host $GPU_FLAG \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /workspace/experiments \ + ${DEEPSTREAM_IMAGE:-nvcr.io/nvidia/deepstream:9.0-triton-multiarch} \ + deepstream-test5-app -c config_deepstream.txt \ No newline at end of file diff --git a/deepstream-tracker-3d-multi-view/scripts/test_12cam_ib.sh b/deepstream-tracker-3d-multi-view/scripts/test_12cam_ib.sh new file mode 100755 index 0000000..da4e97b --- /dev/null +++ b/deepstream-tracker-3d-multi-view/scripts/test_12cam_ib.sh @@ -0,0 +1,75 @@ +# Set dataset, model, experiment, and repo directories +export DATASET_DIR=$PWD/datasets/mtmc_12cam/ +export EXPERIMENT_DIR=$PWD/experiments/inference_builder/12cam +export MODEL_REPO=$PWD/models + +REPO_DIR=$PWD +INFERENCE_BUILDER_DIR=${INFERENCE_BUILDER_DIR:-$HOME/inference_builder} + +# Select detector model: PeopleNetTransformer (default), RTDETR, or PeopleNet2.6.3 +export DETECTOR_MODEL=${DETECTOR_MODEL:-PeopleNetTransformer} + +if [ "$DETECTOR_MODEL" = "RTDETR" ]; then + DETECTOR_CONFIG="config_pgie_rt_detr.txt" + TRACKER_CONFIG="config_tracker.yml" +elif [ "$DETECTOR_MODEL" = "PeopleNet2.6.3" ]; then + DETECTOR_CONFIG="config_pgie_peoplenet.txt" + TRACKER_CONFIG="config_tracker.yml" +else + DETECTOR_CONFIG="config_pgie.txt" + TRACKER_CONFIG="config_tracker_tuned_12cam.yml" +fi + +echo "Using detector model: $DETECTOR_MODEL (detector=$DETECTOR_CONFIG, tracker=$TRACKER_CONFIG)" + +# Set correct GPU flag considering diffent platforms +if docker info | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" +elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" +else + echo "No GPU support found in Docker." + exit 1 +fi + +# Clean and create directories for output +mkdir -p $EXPERIMENT_DIR/infer-kitti-dump +mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump + +# Generate Inference Builder configuration files +source mv3dt_venv/bin/activate +python utils/inference_builder_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --detector-config=$DETECTOR_CONFIG \ + --tracker-config=$TRACKER_CONFIG \ + --output-dir=$EXPERIMENT_DIR + +cp $EXPERIMENT_DIR/nvdsinfer_config.yaml $MODEL_REPO/$DETECTOR_MODEL/ + +# Generate Python package for MV3DT inference flow using Inference Builder +cd $INFERENCE_BUILDER_DIR +source ib_venv/bin/activate +python builder/main.py $EXPERIMENT_DIR/ds_mv3dt.yaml \ + -o builder/samples/mv3dt_app \ + --server-type serverless + +# Launch real-time BEV visualization +cd $REPO_DIR +source mv3dt_venv/bin/activate +python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids & + +# Launch MV3DT pipeline with Inference Builder +docker run --privileged --rm -it --net=host $GPU_FLAG \ + -v $INFERENCE_BUILDER_DIR/builder/samples/mv3dt_app/deepstream-app:/mv3dt_app \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /mv3dt_app \ + inference-builder-mv3dt:latest \ + python3 __main__.py --source-config /workspace/experiments/source_list_static.yaml -s /dev/null diff --git a/deepstream-tracker-3d-multi-view/scripts/test_4cam_ds.sh b/deepstream-tracker-3d-multi-view/scripts/test_4cam_ds.sh new file mode 100755 index 0000000..0e9da4e --- /dev/null +++ b/deepstream-tracker-3d-multi-view/scripts/test_4cam_ds.sh @@ -0,0 +1,65 @@ +# Set dataset, model, and experiment directories +export DATASET_DIR=$PWD/datasets/mtmc_4cam/ +export EXPERIMENT_DIR=$PWD/experiments/deepstream/4cam +export MODEL_REPO=$PWD/models + +# Select detector model: PeopleNetTransformer (default), RTDETR, or PeopleNet2.6.3 +export DETECTOR_MODEL=${DETECTOR_MODEL:-PeopleNetTransformer} + +if [ "$DETECTOR_MODEL" = "RTDETR" ]; then + DETECTOR_CONFIG="config_pgie_rt_detr.txt" + TRACKER_CONFIG="config_tracker.yml" +elif [ "$DETECTOR_MODEL" = "PeopleNet2.6.3" ]; then + DETECTOR_CONFIG="config_pgie_peoplenet.txt" + TRACKER_CONFIG="config_tracker.yml" +else + DETECTOR_CONFIG="config_pgie.txt" + TRACKER_CONFIG="config_tracker.yml" +fi + +echo "Using detector model: $DETECTOR_MODEL (detector=$DETECTOR_CONFIG, tracker=$TRACKER_CONFIG)" + +# Set correct GPU flag considering diffent platforms +if docker info | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" +elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" +else + echo "No GPU support found in Docker." + exit 1 +fi + +# Create directories for output +mkdir -p $EXPERIMENT_DIR/infer-kitti-dump +mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump +mkdir -p $EXPERIMENT_DIR/outVideos + +# Auto-generate DeepStream configuration files +source mv3dt_venv/bin/activate +python utils/deepstream_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --enable-msg-broker \ + --enable-osd \ + --detector-config=$DETECTOR_CONFIG \ + --tracker-config=$TRACKER_CONFIG \ + --config-overrides=override_tracker_4cam.yml \ + --output-dir=$EXPERIMENT_DIR + +# Launch real-time BEV visualization +python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids & + + +# Launch MV3DT pipeline +docker run -t --privileged --rm --net=host $GPU_FLAG \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /workspace/experiments \ + ${DEEPSTREAM_IMAGE:-nvcr.io/nvidia/deepstream:9.0-triton-multiarch} \ + deepstream-test5-app -c config_deepstream.txt diff --git a/deepstream-tracker-3d-multi-view/scripts/test_4cam_ib.sh b/deepstream-tracker-3d-multi-view/scripts/test_4cam_ib.sh new file mode 100755 index 0000000..0cfefd5 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/scripts/test_4cam_ib.sh @@ -0,0 +1,76 @@ +# Set dataset, model, experiment, and repo directories +export DATASET_DIR=$PWD/datasets/mtmc_4cam/ +export EXPERIMENT_DIR=$PWD/experiments/inference_builder/4cam +export MODEL_REPO=$PWD/models + +REPO_DIR=$PWD +INFERENCE_BUILDER_DIR=${INFERENCE_BUILDER_DIR:-$HOME/inference_builder} + +# Select detector model: PeopleNetTransformer (default), RTDETR, or PeopleNet2.6.3 +export DETECTOR_MODEL=${DETECTOR_MODEL:-PeopleNetTransformer} + +if [ "$DETECTOR_MODEL" = "RTDETR" ]; then + DETECTOR_CONFIG="config_pgie_rt_detr.txt" + TRACKER_CONFIG="config_tracker.yml" +elif [ "$DETECTOR_MODEL" = "PeopleNet2.6.3" ]; then + DETECTOR_CONFIG="config_pgie_peoplenet.txt" + TRACKER_CONFIG="config_tracker.yml" +else + DETECTOR_CONFIG="config_pgie.txt" + TRACKER_CONFIG="config_tracker.yml" +fi + +echo "Using detector model: $DETECTOR_MODEL (detector=$DETECTOR_CONFIG, tracker=$TRACKER_CONFIG)" + +# Set correct GPU flag considering diffent platforms +if docker info | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" +elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" +else + echo "No GPU support found in Docker." + exit 1 +fi + +# Create directories for output +mkdir -p $EXPERIMENT_DIR/infer-kitti-dump +mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump + +# Generate Inference Builder configuration files +source mv3dt_venv/bin/activate +python utils/inference_builder_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --detector-config=$DETECTOR_CONFIG \ + --tracker-config=$TRACKER_CONFIG \ + --config-overrides=override_tracker_4cam.yml \ + --output-dir=$EXPERIMENT_DIR + +cp $EXPERIMENT_DIR/nvdsinfer_config.yaml $MODEL_REPO/$DETECTOR_MODEL/ + +# Generate Python package for MV3DT inference flow using Inference Builder +cd $INFERENCE_BUILDER_DIR +source ib_venv/bin/activate +python builder/main.py $EXPERIMENT_DIR/ds_mv3dt.yaml \ + -o builder/samples/mv3dt_app \ + --server-type serverless + +# # Launch real-time BEV visualization +cd $REPO_DIR +source mv3dt_venv/bin/activate +python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids & + +# Launch MV3DT pipeline with Inference Builder +docker run --privileged --rm -it --net=host $GPU_FLAG \ + -v $INFERENCE_BUILDER_DIR/builder/samples/mv3dt_app/deepstream-app:/mv3dt_app \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /mv3dt_app \ + inference-builder-mv3dt:latest \ + python3 __main__.py --source-config /workspace/experiments/source_list_static.yaml -s /dev/null diff --git a/deepstream-tracker-3d-multi-view/scripts/test_custom_2d_tracker_ds.sh b/deepstream-tracker-3d-multi-view/scripts/test_custom_2d_tracker_ds.sh new file mode 100755 index 0000000..0324b07 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/scripts/test_custom_2d_tracker_ds.sh @@ -0,0 +1,46 @@ +# Set dataset, model, and experiment directories +export DATASET_DIR=$PWD/datasets/mtmc_6cam/ +export EXPERIMENT_DIR=$PWD/experiments/deepstream/6cam +export MODEL_REPO=$PWD/models + +# Set correct GPU flag considering diffent platforms +if docker info | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" +elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" +else + echo "No GPU support found in Docker." + exit 1 +fi + +# Create directories for output +mkdir -p $EXPERIMENT_DIR/infer-kitti-dump +mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump +mkdir -p $EXPERIMENT_DIR/outVideos + +# Auto-generate DeepStream configuration files with custom 2D tracker config +source mv3dt_venv/bin/activate +python utils/deepstream_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --enable-msg-broker \ + --enable-osd \ + --tracker-config=config_tracker_2d.yml \ + --output-dir=$EXPERIMENT_DIR + +# Launch real-time BEV visualization +python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids & + +# Launch MV3DT pipeline with custom 2D tracker config +docker run -t --privileged --rm --net=host $GPU_FLAG \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /workspace/experiments \ + ${DEEPSTREAM_IMAGE:-nvcr.io/nvidia/deepstream:9.0-triton-multiarch} \ + deepstream-test5-app -c config_deepstream.txt \ No newline at end of file diff --git a/deepstream-tracker-3d-multi-view/scripts/test_custom_2d_tracker_ib.sh b/deepstream-tracker-3d-multi-view/scripts/test_custom_2d_tracker_ib.sh new file mode 100755 index 0000000..9408700 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/scripts/test_custom_2d_tracker_ib.sh @@ -0,0 +1,58 @@ +# Set dataset, model, experiment, and repo directories +export DATASET_DIR=$PWD/datasets/mtmc_6cam/ +export EXPERIMENT_DIR=$PWD/experiments/inference_builder/6cam +export MODEL_REPO=$PWD/models + +export REPO_DIR=$PWD +export INFERENCE_BUILDER_DIR=${INFERENCE_BUILDER_DIR:-$HOME/inference_builder} + +# Set correct GPU flag considering diffent platforms +if docker info | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" +elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" +else + echo "No GPU support found in Docker." + exit 1 +fi + +# Create directories for output +mkdir -p $EXPERIMENT_DIR/infer-kitti-dump +mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump + +# Generate Inference Builder configuration files with custom 2D tracker config +source mv3dt_venv/bin/activate +python utils/inference_builder_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --tracker-config=config_tracker_2d.yml \ + --output-dir=$EXPERIMENT_DIR + +cp $EXPERIMENT_DIR/nvdsinfer_config.yaml $MODEL_REPO/PeopleNetTransformer/ + +# Generate Python package for MV3DT inference flow using Inference Builder +cd $INFERENCE_BUILDER_DIR +source ib_venv/bin/activate +python builder/main.py $EXPERIMENT_DIR/ds_mv3dt.yaml \ + -o builder/samples/mv3dt_app \ + --server-type serverless + +# Launch real-time BEV visualization +cd $REPO_DIR +source mv3dt_venv/bin/activate +python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids & + +# Launch MV3DT pipeline with Inference Builder and custom 2D tracker config +docker run --privileged --rm -it --net=host $GPU_FLAG \ + -v $INFERENCE_BUILDER_DIR/builder/samples/mv3dt_app/deepstream-app:/mv3dt_app \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /mv3dt_app \ + inference-builder-mv3dt:latest \ + python3 __main__.py --source-config /workspace/experiments/source_list_static.yaml -s /dev/null diff --git a/deepstream-tracker-3d-multi-view/utils/README.md b/deepstream-tracker-3d-multi-view/utils/README.md new file mode 100644 index 0000000..6be7107 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/utils/README.md @@ -0,0 +1,103 @@ +## Python Utility Scripts + +This directory contains Python utility scripts to help with configuration generation and visualization for the DeepStream Multi-View 3D Tracking project. + + +### `deepstream_auto_configurator.py` + +The Deepstream auto-configurator automatically generates DeepStream configuration files based on your dataset. + +**Usage:** +```bash +python deepstream_auto_configurator.py --dataset-dir DATASET_DIR [OPTIONS] +``` + +**Arguments:** +``` +- `--dataset-dir`: Dataset directory containing `videos/` and `camInfo/` subdirectories +- `--output-dir`: Output directory for generated configs (default: `temp_outputs`) +- `--enable-osd`: Enable OSD display sink +- `--enable-file-output`: Enable video file output +- `--enable-msg-broker`: Enable Kafka message broker output +- `--num_vision_neighbor`: Number of vision neighbors per camera +- `--tracker-config`: Base tracker configuration file (default: `config_tracker.yml`) +- `--config-overrides`: YAML file containing section overrides for tracker configuration +``` + + +### `inference_builder_auto_configurator.py` + +The Inference Builder auto-configurator automatically generates Inference Builder config files based on your dataset. + +**Usage:** +```bash +python inference_builder_auto_configurator.py --dataset-dir DATASET_DIR [OPTIONS] +``` + +**Arguments:** +``` +- `--dataset-dir`: Dataset directory containing `videos/` and `camInfo/` subdirectories +- `--output-dir`: Output directory for generated configs (default: `temp_outputs`) +- `--tracker-config`: Base tracker configuration file (default: `config_tracker.yml`) +- `--config-overrides`: YAML file containing section overrides for tracker configuration +``` + + + +### `kafka_bev_visualizer.py` + +Real-time bird's-eye view visualization of 3D tracking data from Kafka streams. + +**Usage:** +```bash +python kafka_bev_visualizer.py [OPTIONS] +``` + +**Arguments:** +``` +- `--dataset-path`: Path to dataset containing map.png and transforms.yml +- `--msgconv-config`: Path to message converter config file +- `--output-path`: Output directory for videos and screenshots +- `--offline`: Run in offline mode to save video from all messages instead of real-time visualization +- `--show-ids`: Show object IDs near trajectory heads +- `--average-multi-cam`: Average trajectory points from multiple cameras for the same object (shows 1 point per object instead of multiple points from different cameras) +``` + +**Interactive Controls (Real-time mode):** +- `q`: Quit application +- `s`: Save current frame as screenshot +- `c`: Clear all trajectories +- `r`: Start/stop recording video + + + +## Additional Utilities + +### `generate_pub_sub_configs.py` + +Generates communication configurations for multi-camera tracking systems, including peer-to-peer relationships and camera neighbor mappings. + +**Usage:** +```bash +python generate_pub_sub_configs.py --deployment_config_path CONFIG_PATH [OPTIONS] +``` + +**Arguments:** +``` +- `--deployment_config_path`: Path to YAML file containing deployment configurations +- `--cam_info_path`: Directory containing camera calibration information +- `--output_path`: Directory to store output configuration files +- `--neighbor_criteria`: Criteria for selecting neighboring cameras + - Format: `"top_N:{N}"` or `"overlap_threshold:{threshold}"` +- `--minimum_object_size`: Minimum object size in pixels for visibility +- `--range_of_interest`: Range of interest in world coordinates + - Format: `"x1,y1,x2,y2"` (min and max corners) +``` + + +**Note:** This script is typically called automatically by the `deepstream_auto_configurator.py` script. + + +### `schema_pb2.py` + +Contains Protocol Buffer schema definitions for message serialization used in Kafka communication. diff --git a/deepstream-tracker-3d-multi-view/utils/deepstream_auto_configurator.py b/deepstream-tracker-3d-multi-view/utils/deepstream_auto_configurator.py new file mode 100644 index 0000000..07fe240 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/utils/deepstream_auto_configurator.py @@ -0,0 +1,651 @@ +#!/usr/bin/env python3 +import os +import argparse +import math +import glob +import re +import shutil +import subprocess +import yaml +import cv2 +from pathlib import Path +from typing import List, Tuple, Dict + +class DeepStreamAutoConfigurator: + def __init__(self, dataset_dir: str, output_dir: str): + self.dataset_dir = Path(dataset_dir) + self.output_dir = Path(output_dir) + + def compute_grid_layout(self, num_videos: int) -> Tuple[int, int]: + if num_videos <= 1: return 1, 1 + elif num_videos == 2: return 1, 2 + elif num_videos <= 4: return 2, 2 + elif num_videos <= 6: return 2, 3 + elif num_videos <= 9: return 3, 3 + elif num_videos <= 12: return 3, 4 + elif num_videos <= 16: return 4, 4 + else: + side = math.ceil(math.sqrt(num_videos)) + return side, side + + def get_video_resolution(self, video_path: str) -> Tuple[int, int]: + cap = cv2.VideoCapture(str(video_path)) + if not cap.isOpened(): + raise RuntimeError(f"Could not open video file: {video_path}") + + width, height = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + cap.release() + + if width <= 0 or height <= 0: + raise RuntimeError(f"Invalid resolution detected: {width}x{height}") + return width, height + + def get_engine_file_from_detector_config(self, detector_config: str, num_videos: int) -> str: + """Extract onnx-file from detector config and generate engine file path.""" + config_path = Path("config_templates") / detector_config + if not config_path.exists(): + return None + + with open(config_path, 'r') as f: + for line in f: + if line.strip().startswith('onnx-file='): + onnx_file = line.strip().split('=', 1)[1] + engine_file = f"{onnx_file}_b{num_videos}_gpu0_fp16.engine" + print(f"Generated engine file: {engine_file}") + return engine_file + return None + + def generate_deepstream_config(self, video_files: List[str], enabled_sinks: List[str], detector_config: str = None) -> str: + template_path = Path("config_templates/config_deepstream.txt") + if not template_path.exists(): + raise FileNotFoundError(f"Template not found: {template_path}") + + with open(template_path, 'r') as f: + content = f.read() + + num_videos = len(video_files) + rows, columns = self.compute_grid_layout(num_videos) + + # Detect video resolution from the first video file + first_video_path = self.dataset_dir / "videos" / video_files[0] + video_width, video_height = self.get_video_resolution(first_video_path) + + print(f"Detected video resolution: {video_width}x{video_height}") + + # Get engine file path from detector config + engine_file_path = self.get_engine_file_from_detector_config(detector_config, num_videos) if detector_config else None + print (f"Detector config: {detector_config}") + print (f"Engine file path: {engine_file_path}") + + # Update source sections and URIs + content = self._update_sources(content, video_files) + + # Simple line-by-line replacements + lines = content.split('\n') + current_section = None + + for i, line in enumerate(lines): + stripped = line.strip() + + if stripped.startswith('[') and stripped.endswith(']'): + current_section = stripped[1:-1] + elif stripped.startswith('batch-size='): + lines[i] = f"batch-size={num_videos}" + elif stripped.startswith('model-engine-file='): + lines[i] = f"model-engine-file={engine_file_path}" if engine_file_path else re.sub(r'_b\d+_', f'_b{num_videos}_', line) + elif stripped.startswith('rows='): + lines[i] = f"rows={rows}" + elif stripped.startswith('columns='): + lines[i] = f"columns={columns}" + elif stripped.startswith('enable=') and current_section in ['sink0', 'sink1', 'sink2', 'sink3']: + lines[i] = f"enable={'1' if current_section in enabled_sinks else '0'}" + # Update streammux section resolution + elif stripped.startswith('width=') and current_section == 'streammux': + lines[i] = f"width={video_width}" + elif stripped.startswith('height=') and current_section == 'streammux': + lines[i] = f"height={video_height}" + # Update tracker section resolution + elif stripped.startswith('tracker-width=') and current_section == 'tracker': + lines[i] = f"tracker-width={video_width}" + elif stripped.startswith('tracker-height=') and current_section == 'tracker': + lines[i] = f"tracker-height={video_height}" + + return '\n'.join(lines) + + def _update_sources(self, content: str, video_files: List[str]) -> str: + lines = content.split('\n') + result = [] + i = 0 + sources_added = 0 + + while i < len(lines): + line = lines[i] + + if line.strip().startswith('[source') and sources_added < len(video_files): + # Keep/add source section + result.append(line) + i += 1 + while i < len(lines) and not (lines[i].strip().startswith('[') and lines[i].strip().endswith(']')): + if lines[i].strip().startswith('uri='): + result.append(f"uri=file:///workspace/inputs/videos/{video_files[sources_added]}") + else: + result.append(lines[i]) + i += 1 + sources_added += 1 + continue + elif line.strip().startswith('[source') and sources_added >= len(video_files): + # Skip extra source sections + i += 1 + while i < len(lines) and not (lines[i].strip().startswith('[') and lines[i].strip().endswith(']')): + i += 1 + continue + elif line.strip().startswith('[streammux]'): + # Add missing sources before streammux + while sources_added < len(video_files): + result.extend([ + "", f"[source{sources_added}]", "type=3", "enable=1", + "cudadec-memtype=0", "gpu-id=0", "num-sources=1", + f"uri=file:///workspace/inputs/videos/{video_files[sources_added]}" + ]) + sources_added += 1 + + # Add blank line before streammux section + result.append("") + result.append(line) + else: + result.append(line) + i += 1 + + return '\n'.join(result) + + def generate_msgconv_config(self, video_files: List[str]) -> str: + """Generate message converter configuration based on video files.""" + lines = [] + + # Sort video files to ensure consistent ordering + sorted_video_files = sorted(video_files) + + for i, video_file in enumerate(sorted_video_files, 1): + # Use index-based camera IDs (1, 2, 3, 4...) + camera_id = i + + lines.extend([ + f"[sensor{i-1}]", + "enable=1", + "type=Camera", + f"id=Camera{camera_id}", + "" # Empty line between sections + ]) + + # Add extra empty line at the end to match template format + lines.append("") + + return '\n'.join(lines) + + def generate_tracker_config(self, video_files: List[str], calib_files: List[str], config_overrides: str = None, tracker_config: str = None) -> str: + # Use specified tracker config or default to config_tracker.yml + tracker_template = tracker_config or "config_tracker.yml" + template_path = Path("config_templates") / tracker_template + if not template_path.exists(): + raise FileNotFoundError(f"Template not found: {template_path}") + + with open(template_path, 'r') as f: + content = f.read() + + # check if this is a 2d config + is_2d_config = self._is_2d_tracker_config(content) + + # Load config overrides if provided + overrides = self._load_section_overrides(config_overrides) + if overrides and not is_2d_config: + print(f"Applying section overrides from: {config_overrides}") + content = self._apply_section_overrides(content, overrides) + + # Augment 2D config if needed + if is_2d_config: + print("Detected 2D tracker config - adding ObjectModelProjection, MultiViewAssociator, and Communicator sections") + # Always add default multiview sections first + content = self._add_multiview_sections(content, calib_files) + # Then apply any user overrides on top + if overrides: + content = self._apply_section_overrides(content, overrides) + + lines = content.split('\n') + num_videos = len(video_files) + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + + if stripped.startswith('cameraModelFilepath:'): + i += 1 + # Remove existing entries + start_idx = i + while i < len(lines) and lines[i].startswith(' - '): + i += 1 + # Delete the old entries + del lines[start_idx:i] + i = start_idx # Reset index to where we deleted + + # Add new camera paths - one for each video file + print(f"Mapping calibration files to {num_videos} videos:") + for j, video_file in enumerate(video_files): + # Try to find corresponding calibration file + calib_file = self._find_matching_calib_file(video_file, calib_files, j) + print(f" Video {j+1}: {video_file} -> {calib_file}") + lines.insert(i, f" - /workspace/inputs/camInfo/{calib_file}") + i += 1 + continue + if stripped.startswith('stateEstimatorType:') and is_2d_config: + old_value = lines[i].split(':')[1].strip() + if old_value != '3': + lines[i] = " stateEstimatorType: 3" + print(f"stateEstimatorType updated to 3") + elif stripped.startswith('visualTrackerType:') and is_2d_config: + old_value = lines[i].split(':')[1].strip() + if old_value != '2': + lines[i] = " visualTrackerType: 2" + print(f"visualTrackerType updated to 2") + + i += 1 + + return '\n'.join(lines) + + def extract_camera_ids(self, video_files: List[str]) -> List[int]: + """Extract camera IDs by using sorted index + 1.""" + sorted_files = sorted(video_files) + return [i + 1 for i in range(len(sorted_files))] + + def generate_deployment_config(self, cam_ids: List[int]) -> str: + """Generate deployment config file for production mode.""" + deployment_config = { + 'mqtt_broker_per_instance': ["127.0.0.1:1883"], + 'topic_template': "/trck/cam%d", + 'ds_instance_cam_assignment': [cam_ids], + 'ds_instance_gpu_assignment': [0] + } + + # Save to temporary file + deployment_config_path = self.output_dir / "deployment_config.yml" + with open(deployment_config_path, 'w') as f: + yaml.dump(deployment_config, f, default_flow_style=False) + + print(f"Generated deployment config: {deployment_config_path}") + return str(deployment_config_path) + + def _is_2d_tracker_config(self, content: str) -> bool: + """Check if the tracker config is missing 3D multi-view sections.""" + return not any(section in content for section in [ + 'ObjectModelProjection:', 'MultiViewAssociator:', 'Communicator:' + ]) + + def _load_section_overrides(self, override_file: str) -> Dict: + """Load section overrides from a YAML file.""" + if not override_file: + return {} + + override_path = Path(override_file) + if not override_path.exists(): + # Try relative to config_templates directory + override_path = Path("config_templates") / override_file + if not override_path.exists(): + print(f"Warning: Override file not found: {override_file}") + return {} + + try: + with open(override_path, 'r') as f: + return yaml.safe_load(f) or {} + except Exception as e: + print(f"Error loading override file {override_path}: {e}") + return {} + + def _apply_section_overrides(self, content: str, overrides: Dict) -> str: + """Apply section overrides to the config content.""" + if not overrides: + return content + + lines = content.split('\n') + result_lines = [] + i = 0 + + while i < len(lines): + line = lines[i] + section_name = None + + # Check if this line starts a section that has an override + stripped = line.strip() + if stripped.endswith(':') and not stripped.startswith('-') and not stripped.startswith('#'): + potential_section = stripped[:-1] # Remove the colon + if potential_section in overrides: + section_name = potential_section + + if section_name: + # Skip the original section + result_lines.append(f"# Original {section_name} section replaced by override") + i += 1 + # Skip all lines until next section + while i < len(lines): + next_line = lines[i].strip() + # Stop if we hit another top-level section (no leading spaces and ends with :) + if (next_line.endswith(':') and not next_line.startswith('-') and + not next_line.startswith('#') and not lines[i].startswith(' ')): + break + i += 1 + + # Add the override section + result_lines.append(f"{section_name}:") + section_data = overrides[section_name] + if isinstance(section_data, dict): + for key, value in section_data.items(): + if isinstance(value, list): + result_lines.append(f" {key}:") + for item in value: + result_lines.append(f" - {item}") + else: + result_lines.append(f" {key}: {value}") + else: + result_lines.append(line) + i += 1 + + return '\n'.join(result_lines) + + def _add_multiview_sections(self, content: str, calib_files: List[str]) -> str: + """Add ObjectModelProjection, MultiViewAssociator, Communicator, and PoseEstimator sections to 2D config.""" + lines = content.split('\n') + + # Check which sections already exist + has_object_model_projection = any('ObjectModelProjection:' in line for line in lines) + has_multi_view_associator = any('MultiViewAssociator:' in line for line in lines) + has_communicator = any('Communicator:' in line for line in lines) + has_pose_estimator = any('PoseEstimator:' in line for line in lines) + + # Append to the bottom of the config + insertion_point = len(lines) + + # Add ObjectModelProjection section if it doesn't exist + if not has_object_model_projection: + print("Adding ObjectModelProjection section") + object_model_sections = [ + 'ObjectModelProjection:', + ' outputFootLocation: 1', + ' outputVisibility: 1', + ' outputConvexHull: 0', + ' objectModelType: 0', + ' cameraModelFilepath:' + ] + + # Add camera model file paths + for calib_file in calib_files: + object_model_sections.append(f' - /workspace/inputs/camInfo/{calib_file}') + + # Insert ObjectModelProjection section + for j, section_line in enumerate(object_model_sections): + lines.insert(insertion_point + j, section_line) + insertion_point += len(object_model_sections) + + # Add MultiViewAssociator section if it doesn't exist + if not has_multi_view_associator: + print("Adding MultiViewAssociator section") + multiview_sections = [ + 'MultiViewAssociator:', + ' multiViewAssociatorType: 1', + ' enableLatePeerReAssoc: 1', + ' enableIDCorrection: 1', + ' enableSeeThrough: 1', + ' enableMsgSync: 1', + ' maxPeerTrackletSize: 50', + ' recentlyActiveAge: 178', + ' minCommonFrames4MatchScore: 2', + ' maxPeerToPredDistance4Fusion: 1.35', + ' minPeerVisibility4Fusion: 0.15', + ' minPeerTrackletMatchScore: 0.48', + ' maxTrackletMatchingTimeSearchRange: 1', + ' maxPeerFrameDiff4NoDet: 2', + ' communicatorInitSleepTime: 0', + ] + + # Insert MultiViewAssociator section + for j, section_line in enumerate(multiview_sections): + lines.insert(insertion_point + j, section_line) + insertion_point += len(multiview_sections) + + # Add Communicator section if it doesn't exist + if not has_communicator: + print("Adding Communicator section") + communicator_sections = [ + 'Communicator:', + ' communicatorType: 2', + ' pubSubInfoConfigPath: /workspace/experiments/pub_sub_info_config_0.yml', + ' mqttProtoAdaptorConfigPath: /workspace/experiments/config_mqtt.txt', + ] + + # Insert Communicator section + for j, section_line in enumerate(communicator_sections): + lines.insert(insertion_point + j, section_line) + insertion_point += len(communicator_sections) + + # Add PoseEstimator section if it doesn't exist + if not has_pose_estimator: + print("Adding PoseEstimator section") + pose_estimator_sections = [ + 'PoseEstimator:', + ' poseEstimatorType: 1', + ' useVPICropScaler: 1', + ' batchSize: 1', + ' workspaceSize: 1000', + ' inferDims: [3, 256, 192]', + ' networkMode: 1', + ' inputOrder: 0', + ' colorFormat: 0', + ' offsets: [123.6750, 116.2800, 103.5300]', + ' netScaleFactor: 0.00392156', + ' onnxFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx', + ' modelEngineFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx_b1_gpu0_fp16.engine', + ' poseInferenceInterval: -1', + '' # Empty line + ] + + # Insert PoseEstimator section + for j, section_line in enumerate(pose_estimator_sections): + lines.insert(insertion_point + j, section_line) + + return '\n'.join(lines) + + def _find_matching_calib_file(self, video_file: str, calib_files: List[str], index: int) -> str: + """Find the corresponding calibration file for a video file.""" + # Remove extension from video file to get base name + video_base = os.path.splitext(video_file)[0] + + # Try to find exact match by replacing video extension with .yml + yml_candidate = video_base + ".yml" + if yml_candidate in calib_files: + return yml_candidate + + # Try to find match by extracting camera number/ID from video filename + video_match = re.search(r'[Cc]am(\d+)', video_file) + if video_match: + cam_id = video_match.group(1) + # Look for calibration files with the same camera ID + for calib_file in calib_files: + calib_match = re.search(r'[Cc]am(\d+)', calib_file) + if calib_match and calib_match.group(1) == cam_id: + return calib_file + + # Fallback: use index-based matching (sorted order) + if index < len(calib_files): + return calib_files[index] + + # Last resort: generate expected filename based on pattern + # If we have calib files, try to follow their naming pattern + if calib_files: + # Get the pattern from the first calibration file + first_calib = calib_files[0] + calib_match = re.search(r'(.+[Cc]am)(\d+)(\.yml)$', first_calib) + if calib_match: + prefix, _, suffix = calib_match.groups() + cam_id = str(index + 1).zfill(len(calib_match.group(2))) + expected_file = f"{prefix}{cam_id}{suffix}" + print(f" Warning: Expected calibration file {expected_file} not found!") + return expected_file + + # Final fallback: generate a generic name + expected_file = f"camera_{index + 1:03d}.yml" + print(f" Warning: Generated fallback calibration filename {expected_file}") + return expected_file + + def generate_pub_sub_config(self, dataset_dir: str, video_files: List[str], num_vision_neighbor: int = 3, use_debug_communicator: bool = False): + """Generate pub_sub_info_config_0.yml using the existing script.""" + cam_ids = self.extract_camera_ids(video_files) + cam_subset = ','.join([str(cam_id) for cam_id in cam_ids]) + + # Check if the script exists + script_path = Path("utils/generate_pub_sub_configs.py") + if not script_path.exists(): + print(f"Error: Script not found at {script_path}") + print(f"Current working directory: {os.getcwd()}") + return + + command = [ + "python", str(script_path), + "--cam_info_path", os.path.join(dataset_dir, "camInfo"), + "--neighbor_criteria", f"top_N:{num_vision_neighbor}", + "--output_path", str(self.output_dir) + ] + + if use_debug_communicator: + command.extend(["--use_debug_communicator", "--cam_subset", cam_subset]) + else: + # Generate deployment config for production mode + deployment_config_path = self.generate_deployment_config(cam_ids) + command.extend(["--deployment_config_path", deployment_config_path]) + + try: + print(f"Generating pub_sub config for cameras: {cam_subset} (vision neighbors: {num_vision_neighbor})") + print(f"Command: {' '.join(command)}") + result = subprocess.run(command, check=True, capture_output=True, text=True) + + # Check if file was actually created + pub_sub_file = self.output_dir / "pub_sub_info_config_0.yml" + if pub_sub_file.exists(): + print(f"Generated: {pub_sub_file}") + else: + print(f"Warning: pub_sub_info_config_0.yml not found at {pub_sub_file}") + print(f"Command stdout: {result.stdout}") + print(f"Command stderr: {result.stderr}") + except subprocess.CalledProcessError as e: + print(f"Error: Failed to generate pub_sub config: {e}") + print(f"Command output: {e.stdout}") + print(f"Command error: {e.stderr}") + except Exception as e: + print(f"Unexpected error during pub_sub generation: {e}") + + def generate_configs(self, enabled_sinks: List[str] = None, config_overrides: str = None, tracker_config: str = None, detector_config: str = None) -> Dict[str, str]: + if enabled_sinks is None: + enabled_sinks = ['sink0'] + + # Detect MP4 video files + video_files = sorted([os.path.basename(f) for f in glob.glob(str(self.dataset_dir / "videos" / "*.mp4"))]) + if not video_files: + raise ValueError(f"No MP4 files found in {self.dataset_dir}/videos") + + # Detect YML calibration files + calib_files = sorted([os.path.basename(f) for f in glob.glob(str(self.dataset_dir / "camInfo" / "*.yml"))]) + + print(f"Found {len(video_files)} videos, {len(calib_files)} calibration files") + + # Generate pub_sub config (will be called from main with num_peers) + # Note: num_peers will be passed from main function + + return { + 'config_deepstream.txt': self.generate_deepstream_config(video_files, enabled_sinks, detector_config), + 'config_tracker.yml': self.generate_tracker_config(video_files, calib_files, config_overrides, tracker_config), + 'config_msgconv.txt': self.generate_msgconv_config(video_files) + } + + def save_configs(self, configs: Dict[str, str]): + self.output_dir.mkdir(parents=True, exist_ok=True) + for filename, content in configs.items(): + path = self.output_dir / filename + with open(path, 'w') as f: + f.write(content) + print(f"Generated: {path}") + + def copy_static_configs(self, detector_config: str = None): + """Copy static config files from templates to output directory.""" + self.output_dir.mkdir(parents=True, exist_ok=True) + + # copy mqtt config + src_path = Path('config_templates') / 'config_mqtt.txt' + dst_path = self.output_dir / 'config_mqtt.txt' + if src_path.exists(): + shutil.copy2(src_path, dst_path) + print(f"Copied: {dst_path}") + else: + print(f"Warning: Template file not found: {src_path}") + + # copy detector config + src_path = Path('config_templates') / detector_config + dst_path = self.output_dir / 'config_pgie.txt' + if src_path.exists(): + shutil.copy2(src_path, dst_path) + print(f"Copied: {dst_path}") + else: + print(f"Warning: Template file not found: {src_path}") + + +def main(): + parser = argparse.ArgumentParser(description='DeepStream Auto-Configurator') + parser.add_argument('--dataset-dir', default='datasets/mtmc_4cam', help='Dataset directory with videos/ and camInfo/') + parser.add_argument('--output-dir', default='temp_outputs', help='Output directory') + parser.add_argument('--enable-osd', action='store_true', help='Enable OSD sink (sink1 - EglSink)') + parser.add_argument('--enable-file-output', action='store_true', help='Enable video file output (sink2 - MP4)') + parser.add_argument('--enable-msg-broker', action='store_true', help='Enable message broker output (sink3 - Kafka)') + parser.add_argument('--num_vision_neighbor', type=int, default=None, help='Number of vision neighbors per camera') + parser.add_argument('--use_debug_communicator', action='store_true', help='Use debug communicator for pub_sub config') + parser.add_argument('--config-overrides', type=str, help='YAML file with section overrides (e.g., override_tracker_4cam.yml, override_tracker_12cam.yml)') + parser.add_argument('--tracker-config', type=str, default='config_tracker.yml', help='Tracker configuration template (e.g., config_tracker_2d.yml)') + parser.add_argument('--detector-config', type=str, default='config_pgie.txt', help='Detector configuration template (e.g., config_pgie.txt, config_pgie_rt_deter.txt)') + + args = parser.parse_args() + + enabled_sinks = ['sink0'] # sink0 always enabled (fake sink, type=1) + if args.enable_osd: enabled_sinks.append('sink1') # sink1 (OSD/EglSink, type=2) + if args.enable_file_output: enabled_sinks.append('sink2') # sink2 (MP4 file output, type=3) + if args.enable_msg_broker: enabled_sinks.append('sink3') # sink3 (Kafka message broker, type=6) + + try: + configurator = DeepStreamAutoConfigurator(args.dataset_dir, args.output_dir) + + # Get video files for pub_sub generation + video_files = sorted([os.path.basename(f) for f in glob.glob(str(Path(args.dataset_dir) / "videos" / "*.mp4"))]) + if args.num_vision_neighbor is None: + args.num_vision_neighbor = len(video_files) - 1 + configurator.generate_pub_sub_config(args.dataset_dir, video_files, args.num_vision_neighbor, args.use_debug_communicator) + + configs = configurator.generate_configs(enabled_sinks, args.config_overrides, args.tracker_config, args.detector_config) + configurator.save_configs(configs) + configurator.copy_static_configs(args.detector_config) + + # Get video count for summary + num_videos = len([f for f in glob.glob(str(Path(args.dataset_dir) / "videos" / "*.mp4"))]) + rows, columns = configurator.compute_grid_layout(num_videos) + + print(f"✅ Generated configs for {num_videos} videos ({rows}x{columns} grid)") + print(f"📁 Output: {args.output_dir}") + print(f"🔧 Sinks: {', '.join(enabled_sinks)}") + print(f"🤝 Vision neighbors: {args.num_vision_neighbor}") + print(f"📋 Generated files:") + print(f" - config_deepstream.txt (main pipeline config)") + print(f" - config_tracker.yml (3D tracker config)") + print(f" - config_msgconv.txt (message converter config)") + print(f" - pub_sub_info_config_0.yml (communication config)") + print(f"\n🚀 Run with DeepStream container using the generated configs") + + except Exception as e: + print(f"❌ Error: {e}") + return 1 + + return 0 + +if __name__ == "__main__": + exit(main()) \ No newline at end of file diff --git a/deepstream-tracker-3d-multi-view/utils/generate_pub_sub_configs.py b/deepstream-tracker-3d-multi-view/utils/generate_pub_sub_configs.py new file mode 100644 index 0000000..dddd927 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/utils/generate_pub_sub_configs.py @@ -0,0 +1,267 @@ +import argparse +import numpy as np +import tqdm +import yaml +import os +import cv2 +import glob +import re +from os.path import join +METERS_TO_FEET = 3.28084 + +def get_camera_fov_mask(cam_calib, num_pix, range_of_interest=None): + P, Q, K, R, _, pos, end, height = cam_calib + cx, cy = K[0, 2], K[1, 2] + cam_h, cam_w = int(cy * 2), int(cx * 2) + limits_ov = range_of_interest + # print ("range_of_interests", limits_ov[0, 0, 0], limits_ov[1, 0, 0], limits_ov[0, 0, 1], limits_ov[1, 0, 1]) + x_ov = np.arange(np.round(limits_ov[0]), np.round(limits_ov[2])) + y_ov = np.arange(np.round(limits_ov[1]), np.round(limits_ov[3])) + xy_ov = np.array(np.meshgrid(x_ov, y_ov), dtype=float).reshape(2, -1) + # Create 3D points in z=0, z=h/2, z=h + xyz_0 = np.vstack([xy_ov, np.zeros(xy_ov.shape[1])]).T.reshape(len(x_ov), len(y_ov), 3) + xyz_h = np.vstack([xy_ov, height * np.ones(xy_ov.shape[1])]).T.reshape(len(x_ov), len(y_ov), 3) + # Project to the image plane + xy0_cam = cv2.perspectiveTransform(xyz_0, P).reshape(-1, 2).T + xyh_cam = cv2.perspectiveTransform(xyz_h, P).reshape(-1, 2).T + # Create the mask + mask1 = (xy0_cam[0] > 0) & (xy0_cam[0] < cam_w) + mask2 = (xy0_cam[1] > 0) & (xy0_cam[1] < cam_h) + mask3 = np.linalg.norm(xyh_cam - xy0_cam, ord=np.inf, axis=0) > num_pix # object size: OK + mask4 = ((xy_ov.T - pos.T) @ (end - pos)).flatten() > 0 # direction: OK + # Update the mask + mask = mask1 & mask2 & mask3 & mask4 + return mask + +def load_and_process_camera_matrices(cam_info_path): + # Load all .yml and .yaml files + cam_files = glob.glob(os.path.join(cam_info_path, "*.yml")) + glob.glob(os.path.join(cam_info_path, "*.yaml")) + + cam_matrices = {} + for idx, cam_file in enumerate(sorted(cam_files)): + cam = idx + 1 + + with open(cam_file, 'r') as file: + yaml_data = yaml.safe_load(file) + if isinstance(yaml_data['modelInfo'], list): + heights = [yaml_data['modelInfo'][i]['height'] for i in range(len(yaml_data['modelInfo']))] + height = max(heights) * METERS_TO_FEET # meters to feet (OV) + else: + height = yaml_data['modelInfo']["height"] * METERS_TO_FEET # meters to feet (OV) + P = np.array(yaml_data["projectionMatrix_3x4_w2p"]).reshape(3, 4) + Q = np.linalg.pinv(P) + K, R, t, _, _, _, _ = cv2.decomposeProjectionMatrix(P) + K = K / K[2, 2] + # Translation vector in OV (-R.t @ t) + pos = t[:2] / t[-1] + # point at 3 OV units (feet) in front of the camera + end = pos + R[-1:, :2].T * (3 / np.linalg.norm(R[-1, :2])) + # pos_px = cv2.perspectiveTransform(pos.reshape(1, 1, 2), T_ov2px).reshape(2) + # end_px = cv2.perspectiveTransform(end.reshape(1, 1, 2), T_ov2px).reshape(2) + cam_matrices[cam] = P, Q, K, R, t, pos, end, height + return cam_matrices + + +def get_overlap_of_2_masks(mask1, mask2): + overlap = np.logical_and(mask1, mask2) + overlap_count = np.sum(overlap) + mask1_ratio = overlap_count / np.sum(mask1) + mask2_ratio = overlap_count / np.sum(mask2) + return mask1_ratio, mask2_ratio, overlap_count + + +def get_overlap_matrix(cam_matrices, minimum_object_size, range_of_interest): + overlap_matrix = {} + masks = {} + + # Generate masks for all cameras + for cam in tqdm.tqdm(cam_matrices, desc="Generating masks"): + mask = get_camera_fov_mask(cam_matrices[cam], num_pix=minimum_object_size, range_of_interest=range_of_interest) + masks[cam] = mask + + # Calculate overlap ratios for all camera pairs + for cam1 in tqdm.tqdm(cam_matrices, desc="Calculating overlaps"): + overlap_matrix[cam1] = {} + mask1 = masks[cam1] + for cam2 in cam_matrices: + if cam1 == cam2: continue + mask2 = masks[cam2] + mask1_ratio, mask2_ratio, _ = get_overlap_of_2_masks(mask1, mask2) + overlap_matrix[cam1][cam2] = mask1_ratio + + return overlap_matrix + + +def get_subscription_map(overlap_matrix, criteria): + criteria_type, value = criteria.split(':') + subscription_map = {} + if criteria_type == 'top_N': + N = int(value) + for cam in overlap_matrix: + subscription_map[cam] = [] + neighbors = list(overlap_matrix[cam].keys()) + top_cam_idxs = np.argpartition([overlap_matrix[cam][nei] for nei in neighbors], -N)[-N:].tolist() + top_cams = [neighbors[i] for i in top_cam_idxs] + subscription_map[cam] = top_cams + + + elif criteria_type == 'overlap_threshold': + threshold = float(value) + for cam in overlap_matrix: + subscription_map[cam] = [] + for neighbor in overlap_matrix[cam]: + if overlap_matrix[cam][neighbor] >= threshold: + subscription_map[cam].append(neighbor) + + return subscription_map + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument( + '--deployment_config_path', + type=str, + default=join(os.getenv("HOME"), "Documents", "deepstream", "workspace", "mqtt", "config_templates", "deployment_configs", "repro_1ds_100cam_orig_mtmc.yml"), + help='Path to yaml file containing deployment related configs' + ) + + parser.add_argument( + '--use_debug_communicator', + action='store_true', + help='Use debug communicator' + ) + + parser.add_argument( + '--cam_info_path', + type=str, + default=join(os.getenv("PWD"), "camInfo"), + help='Directory containing camera calibration info (intrinsic & extrinsic params)' + ) + + parser.add_argument( + '--minimum_object_size', + type=int, + default=150, + help='Number of pixels (in height) to consider an object visible when rendering FOV' + ) + + parser.add_argument( + '--neighbor_criteria', + type=str, + default='overlap_threshold:%f' % (2 / (1920 * 1080)), # 'top_N:3', # 'overlap_threshold:%f' % (2 / (1920 * 1080)), + help='Format: "top_N:{N}" or "overlap_threshold:{thres}". Determines neighbor selection method' + ) + + parser.add_argument( + '--output_path', + type=str, + default='./peer_configs', + help='Directory to store output peer cam config files' + ) + + parser.add_argument( + '--range_of_interest', + type=str, + default=None, + help='Range of interest of world plane in format "x1,y1,x2,y2" where (x1,y1) is min corner and (x2,y2) is max corner' + ) + + return parser.parse_args() + +if __name__ == '__main__': + args = parse_args() + if args.use_debug_communicator: + # For debug mode, use all cameras found in directory + cam_matrices = load_and_process_camera_matrices(args.cam_info_path) + cam_subset = list(cam_matrices.keys()) + else: + # Load deployment config and camera matrices + with open(args.deployment_config_path, 'r') as f: + deployment_config = yaml.safe_load(f) + cam_subset = [cam for cam_list in deployment_config['ds_instance_cam_assignment'] for cam in cam_list] + cam2instance = {} + for instance_id, cam_list in enumerate(deployment_config['ds_instance_cam_assignment']): + for cam in cam_list: + cam2instance[cam] = instance_id + + # Load all cameras, then filter to cam_subset + cam_matrices = load_and_process_camera_matrices(args.cam_info_path) + + # Filter cam_matrices to only include cameras in cam_subset + filtered_cam_matrices = {cam: cam_matrices[cam] for cam in cam_subset if cam in cam_matrices} + + # Parse range of interest + if args.range_of_interest: + x1, y1, x2, y2 = map(float, args.range_of_interest.split(',')) + range_of_interest_ov = np.array([x1, y1, x2, y2], dtype=float) + else: + range_padding = 100 + cam_poses = [filtered_cam_matrices[cam][5] for cam in filtered_cam_matrices] + min_x = min([pose[0][0] for pose in cam_poses]) + max_x = max([pose[0][0] for pose in cam_poses]) + min_y = min([pose[1][0] for pose in cam_poses]) + max_y = max([pose[1][0] for pose in cam_poses]) + range_of_interest_ov = np.array([min_x - range_padding, min_y - range_padding, max_x + range_padding, max_y + range_padding], dtype=float) + + # print (filtered_cam_matrices) + # Calculate overlap matrix for all cameras + overlap_matrix = get_overlap_matrix(filtered_cam_matrices, args.minimum_object_size, range_of_interest_ov) + # for cam in overlap_matrix: + # print (cam, overlap_matrix[cam]) + + # Parse neighbor criteria + subscription_map = get_subscription_map(overlap_matrix, args.neighbor_criteria) + subscription_map = "" + print ('subscription_map:', subscription_map) + + subscription_map = get_subscription_map(overlap_matrix, args.neighbor_criteria) + for cam in subscription_map: + print (" %d:" % (cam), subscription_map[cam]) + + # # sudo apt install graphviz + # # pip install graphviz + # import graphviz + # dot = graphviz.Digraph(comment='Camera Subscriptions') + # nodes_expanded = set() + # nodes_to_expand = set([27]) + # while nodes_to_expand: + # node = nodes_to_expand.pop() + # nodes_expanded.add(node) + # for nei in subscription_map[node]: + # if nei not in nodes_expanded: + # nodes_to_expand.add(nei) + # dot.edge(str(node), str(nei)) + # dot.render('camera_subscriptions', format='png') + + # Get average number of neighbors + num_neighbors = np.mean([len(subscription_map[cam]) for cam in subscription_map]) + print ('Average number of neighbors:', num_neighbors) + + + if args.use_debug_communicator: + peer_config = {'camIds': cam_subset, 'subToCamIds': []} + for cam in cam_subset: + peer_config['subToCamIds'].append(subscription_map[cam]) + if not os.path.exists(args.output_path): + os.makedirs(args.output_path) + with open(os.path.join(args.output_path, 'pub_sub_info_config_0.yml'), 'w') as f: + yaml.dump(peer_config, f, default_flow_style=False) + else: + # Generate pub_sub_info_config.yml files + for instance_id, cam_list in enumerate(deployment_config['ds_instance_cam_assignment']): + instance_config = {"pubBrokerTopicStr": [], "subPeerBrokerTopicStrs": []} + for cam_idx, cam in enumerate(cam_list): + # cam_url = deployment_config['cam_url_map'][cam] + cam_url = deployment_config['mqtt_broker_per_instance'][instance_id] + cam_topic = deployment_config['topic_template'] % cam + instance_config["pubBrokerTopicStr"].append(cam_url + ';' + cam_topic) + instance_config["subPeerBrokerTopicStrs"].append([]) + for nei in subscription_map[cam]: + nei_instance_id = cam2instance[nei] + nei_url = deployment_config['mqtt_broker_per_instance'][nei_instance_id] + nei_topic = deployment_config['topic_template'] % nei + instance_config["subPeerBrokerTopicStrs"][cam_idx].append(nei_url + ';' + nei_topic) + if not os.path.exists(args.output_path): + os.makedirs(args.output_path) + with open(os.path.join(args.output_path, f"pub_sub_info_config_{instance_id}.yml"), 'w') as f: + yaml.dump(instance_config, f, default_flow_style=False) diff --git a/deepstream-tracker-3d-multi-view/utils/inference_builder_auto_configurator.py b/deepstream-tracker-3d-multi-view/utils/inference_builder_auto_configurator.py new file mode 100644 index 0000000..9ea8180 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/utils/inference_builder_auto_configurator.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 + +import argparse +import os +import glob +from pathlib import Path +from typing import List, Dict +import yaml + +# Import the base configurator to reuse functionality +from deepstream_auto_configurator import DeepStreamAutoConfigurator + + +class InferenceBuilderAutoConfigurator(DeepStreamAutoConfigurator): + """ + Auto-configurator for inference-builder based on DeepStream auto-configurator. + Generates ds_mv3dt.yaml and source_list_static.yaml instead of config_deepstream.txt. + """ + + @staticmethod + def parse_model_name(detector_config: str) -> str: + """Extract model name from the onnx-file path in the detector config.""" + config_path = Path("config_templates") / detector_config + if not config_path.exists(): + raise FileNotFoundError(f"Detector config not found: {config_path}") + with open(config_path, 'r') as f: + for line in f: + line = line.strip() + if line.startswith('onnx-file='): + onnx_path = Path(line.split('=', 1)[1]) + return onnx_path.parent.name + raise ValueError(f"No onnx-file entry found in {detector_config}") + + def generate_ds_mv3dt_config(self, video_files: List[str], detector_config: str = None) -> str: + """Generate ds_mv3dt.yaml with correct max_batch_size and model name.""" + template_path = Path("config_templates/ds_mv3dt.yaml") + if not template_path.exists(): + raise FileNotFoundError(f"Template not found: {template_path}") + + with open(template_path, 'r') as f: + config = yaml.safe_load(f) + + # Detect video resolution from the first video file + first_video_path = self.dataset_dir / "videos" / video_files[0] + video_width, video_height = self.get_video_resolution(first_video_path) + print(f"Detected video resolution: {video_width}x{video_height}") + + # Update configuration + num_videos = len(video_files) + model_config = config['models'][0] + model_config['max_batch_size'] = num_videos + model_config['parameters']['resize_video'] = [video_height, video_width] + + detector_config = detector_config or "config_pgie.txt" + model_name = self.parse_model_name(detector_config) + model_config['name'] = model_name + + # Update tracker resolution + if 'tracker_config' in model_config['parameters']: + model_config['parameters']['tracker_config']['width'] = video_width + model_config['parameters']['tracker_config']['height'] = video_height + + return yaml.dump(config, default_flow_style=False, sort_keys=False) + + def generate_source_list_static(self, video_files: List[str]) -> str: + """Generate source_list_static.yaml based on video files.""" + header = """source-list:""" + + # Generate source entries + source_entries = [] + for i, video_file in enumerate(sorted(video_files), 1): + source_entries.extend([ + f'- uri: "file:///workspace/inputs/videos/{video_file}"', + f' sensor-id: Camera{i}', + f' sensor-name: UniqueSensorName{i}' + ]) + + footer = """source-config: + source-bin: "nvurisrcbin" + properties: + file-loop: false""" + + return '\n'.join([header] + source_entries + [footer]) + + def generate_nvdsinfer_config(self, video_files: List[str], detector_config: str = None) -> str: + """Generate nvdsinfer_config.yaml based on detector config template.""" + detector_config = detector_config or "config_pgie.txt" + template_path = Path("config_templates") / detector_config + if not template_path.exists(): + raise FileNotFoundError(f"Template not found: {template_path}") + + with open(template_path, 'r') as f: + content = f.read() + + num_videos = len(video_files) + + # Simple format conversion: INI to YAML with batch-size adjustment + yaml_lines = [] + current_section = None + + for line in content.split('\n'): + line = line.strip() + if not line or line.startswith('#'): + continue + elif line.startswith('[') and line.endswith(']'): + current_section = line[1:-1] # Remove brackets + yaml_lines.append(f'{current_section}:') + elif '=' in line and current_section: + key, value = line.split('=', 1) + if key == 'batch-size': + yaml_lines.append(f' {key}: {num_videos}') + else: + yaml_lines.append(f' {key}: {value}') + + return '\n'.join(yaml_lines) + + def generate_configs(self, enabled_sinks: List[str] = None, config_overrides: str = None, tracker_config: str = None, detector_config: str = None) -> Dict[str, str]: + """Generate inference-builder configs instead of DeepStream configs.""" + # Detect MP4 video files + video_files = sorted([os.path.basename(f) for f in glob.glob(str(self.dataset_dir / "videos" / "*.mp4"))]) + if not video_files: + raise ValueError(f"No MP4 files found in {self.dataset_dir}/videos") + + # Detect YML calibration files + calib_files = sorted([os.path.basename(f) for f in glob.glob(str(self.dataset_dir / "camInfo" / "*.yml"))]) + + print(f"Found {len(video_files)} videos, {len(calib_files)} calibration files") + + return { + 'ds_mv3dt.yaml': self.generate_ds_mv3dt_config(video_files, detector_config), + 'config_tracker.yml': self.generate_tracker_config(video_files, calib_files, config_overrides, tracker_config), + 'source_list_static.yaml': self.generate_source_list_static(video_files), + 'nvdsinfer_config.yaml': self.generate_nvdsinfer_config(video_files, detector_config), + 'config_msgconv.txt': self.generate_msgconv_config(video_files) + } + + +def main(): + parser = argparse.ArgumentParser(description='Inference Builder Auto-Configurator') + parser.add_argument('--dataset-dir', default='datasets/mtmc_4cam', help='Dataset directory with videos/ and camInfo/') + parser.add_argument('--output-dir', default='temp_outputs', help='Output directory') + parser.add_argument('--config-overrides', type=str, help='YAML file with section overrides (e.g., override_tracker_4cam.yml, override_tracker_12cam.yml)') + parser.add_argument('--tracker-config', type=str, default='config_tracker.yml', help='Tracker configuration template (e.g., config_tracker_2d.yml)') + parser.add_argument('--detector-config', type=str, default='config_pgie.txt', help='Detector configuration template (e.g., config_pgie.txt, config_pgie_rt_detr.txt)') + parser.add_argument('--num_vision_neighbor', type=int, default=None, help='Number of vision neighbors per camera') + parser.add_argument('--use_debug_communicator', action='store_true', help='Use debug communicator for pub_sub config') + + args = parser.parse_args() + + try: + configurator = InferenceBuilderAutoConfigurator(args.dataset_dir, args.output_dir) + + # Generate pub_sub config (reuse from parent class) + video_files = sorted([os.path.basename(f) for f in glob.glob(str(Path(args.dataset_dir) / "videos" / "*.mp4"))]) + if args.num_vision_neighbor is None: + args.num_vision_neighbor = len(video_files) - 1 + configurator.generate_pub_sub_config(args.dataset_dir, video_files, args.num_vision_neighbor, args.use_debug_communicator) + + # Generate inference-builder configs + configs = configurator.generate_configs(config_overrides=args.config_overrides, tracker_config=args.tracker_config, detector_config=args.detector_config) + configurator.save_configs(configs) + configurator.copy_static_configs(args.detector_config) + + # Get video count for summary + num_videos = len(video_files) + + print(f"✅ Generated inference-builder configs for {num_videos} videos") + print(f"📁 Output: {args.output_dir}") + print(f"🤝 Vision neighbors: {args.num_vision_neighbor}") + print(f"🔍 Detector config: {args.detector_config}") + print(f"🐛 Debug communicator: {'enabled' if args.use_debug_communicator else 'disabled'}") + if args.config_overrides: + print(f"🔧 Config overrides: {args.config_overrides}") + print(f"📋 Generated files:") + print(f" - ds_mv3dt.yaml (inference config with max_batch_size: {num_videos})") + print(f" - config_tracker.yml (3D tracker config)") + print(f" - source_list_static.yaml (source configuration)") + print(f" - nvdsinfer_config.yaml (inference engine config with batch_size: {num_videos})") + print(f" - config_msgconv.txt (message converter config)") + print(f" - pub_sub_info_config_0.yml (communication config)") + print(f"\n🚀 Run with inference builder using the generated configs") + + except Exception as e: + print(f"❌ Error: {e}") + return 1 + + return 0 + + +if __name__ == "__main__": + exit(main()) \ No newline at end of file diff --git a/deepstream-tracker-3d-multi-view/utils/kafka_bev_visualizer.py b/deepstream-tracker-3d-multi-view/utils/kafka_bev_visualizer.py new file mode 100644 index 0000000..2a9edc8 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/utils/kafka_bev_visualizer.py @@ -0,0 +1,627 @@ +#!/usr/bin/env python3 + +import os, tkinter as tk, argparse +from datetime import datetime +from kafka import KafkaConsumer +from google.protobuf.json_format import MessageToDict +from schema_pb2 import Frame +from collections import defaultdict +import numpy as np, yaml, cv2, time +from tqdm import tqdm + +def extract_expected_sensors(msgconv_config): + """Extract expected sensor IDs from config_msgconv.txt file.""" + try: + expected_sensors = [] + + with open(msgconv_config, 'r') as f: + lines = f.readlines() + + current_section = None + for line in lines: + line = line.strip() + + # Skip empty lines and comments + if not line or line.startswith('#'): + continue + + # Check for section header [sensorX] + if line.startswith('[sensor') and line.endswith(']'): + current_section = line[1:-1] # Remove brackets + continue + + # Look for id= lines within sensor sections + if current_section and line.startswith('id='): + sensor_id = line.split('=', 1)[1].strip() + if sensor_id: + expected_sensors.append(sensor_id) + + if not expected_sensors: + raise ValueError("No sensor IDs found in msgconv config file") + + return expected_sensors + + except (FileNotFoundError, ValueError) as e: + raise Exception(f"Could not parse msgconv config file {msgconv_config}: {e}") + + +class FrameBuffer: + def __init__(self, expected_sensors=None, timeout=0.5, lookahead_frames=1): + """ + For a frame t to be considered complete, 1 of the 3 conditions must be met: + 1. Messages from all expected sensors are received at frame t + 2. Current time - message sent time of any sensor at frame t > timeout + 3. At least 1 of the future frames (t+lookahead_frames) are received + + The current settings is designed for single-container MV3DT. + For multi-container MV3DT, consider increasing the timeout and lookahead_frames. + """ + + self.frame_data = defaultdict(dict) + self.expected_sensors = expected_sensors or set() + self.timeout = timeout + self.timestamps = {} + self.lookahead_frames = lookahead_frames + + def add_frame(self, frame_id, sensor_id, frame_dict): + try: + frame_id = int(frame_id) + except ValueError: + pass + if frame_id not in self.timestamps: + self.timestamps[frame_id] = time.time() + self.frame_data[frame_id][sensor_id] = frame_dict + if not self.expected_sensors: + self.expected_sensors.add(sensor_id) + + def get_complete_frame(self): + current_time = time.time() + num_expected = len(self.expected_sensors) + # Process frames in ascending order when possible + def _key_fn(x): + try: + return int(x) + except Exception: + return float('inf') + for frame_id in sorted(list(self.frame_data.keys()), key=_key_fn): + sensors = set(self.frame_data[frame_id].keys()) + frame_time = self.timestamps.get(frame_id, current_time) + # Condition 1: full set received for this frame + full_received = (num_expected > 0 and len(sensors) >= num_expected and sensors.issubset(self.expected_sensors)) + # Condition 2: timeout + timed_out = (current_time - frame_time) > self.timeout + # Condition 3: at least 1 of future frame (t+lookahead_frames) received + future_received = False + if num_expected > 0: + try: + base_id = int(frame_id) + future_id = base_id + self.lookahead_frames + future_data = self.frame_data.get(future_id) + if future_data is not None: + future_received = True + except Exception: + pass + if full_received or timed_out or future_received: + # print ('frame_id', frame_id, 'full_received', full_received, 'timed_out', timed_out, 'future_received', future_received) + frame_data = self.frame_data.pop(frame_id) + self.timestamps.pop(frame_id, None) + return frame_id, frame_data + return None, None + + def get_all_complete_frames(self): + complete_frames = [] + for frame_id, sensors_data in self.frame_data.items(): + if len(sensors_data) > 0: + try: + complete_frames.append((int(frame_id), frame_id, sensors_data)) + except (ValueError, TypeError): + complete_frames.append((0, frame_id, sensors_data)) + complete_frames.sort(key=lambda x: x[0]) + return [(frame_id, data) for _, frame_id, data in complete_frames] + + def get_frame(self, frame_id): + """Get frame data for specific frame_id without waiting for conditions, and remove it from buffer""" + if frame_id in self.frame_data: + frame_data = self.frame_data.pop(frame_id) + self.timestamps.pop(frame_id, None) + return frame_id, frame_data + return None, None + +def display_frame(vis_img, frame_id, video_writer): + """Add frame info, display frame, and write to video""" + recording = "REC" if video_writer else "" + info = f"Frame: {frame_id} {recording}" + cv2.putText(vis_img, info, (10, vis_img.shape[0] - 20), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) + + cv2.imshow('Bird-Eye View of Multi-View 3D Tracking', vis_img) + + if video_writer: + video_writer.write(vis_img) + +def load_map_and_transforms(dataset_path): + """Load map image and transformation matrix from dataset""" + map_path = os.path.join(dataset_path, 'map.png') + transforms_path = os.path.join(dataset_path, 'transforms.yml') + + with open(transforms_path, 'r') as f: + transforms = yaml.safe_load(f) + T_ov2px = np.array(transforms['T_ov2px']).reshape(3, 3) + + map_img = cv2.imread(map_path) + if map_img is None: + raise FileNotFoundError(f"Map image not found at {map_path}") + + return map_img, T_ov2px + +def setup_map_scaling(map_img, target_width_ratio=0.8, target_height_ratio=0.8): + """Setup map scaling based on screen size""" + root = tk.Tk() + screen_width, screen_height = root.winfo_screenwidth(), root.winfo_screenheight() + root.destroy() + + target_width = int(screen_width * target_width_ratio) + target_height = int(screen_height * target_height_ratio) + + map_height, map_width = map_img.shape[:2] + scale = min(target_width / map_width, target_height / map_height) + new_width, new_height = int(map_width * scale), int(map_height * scale) + + map_img_resized = cv2.resize(map_img, (new_width, new_height)) + scale_matrix = np.array([[scale, 0, 0], [0, scale, 0], [0, 0, 1]]) + + return map_img_resized, scale_matrix, new_width, new_height + +def create_kafka_consumer(group_id, consumer_timeout_ms=None): + """Create and configure Kafka consumer""" + config = { + 'bootstrap_servers': 'localhost:9092', + 'auto_offset_reset': 'earliest', + 'value_deserializer': lambda x: x, + 'group_id': group_id, + 'enable_auto_commit': True + } + + if consumer_timeout_ms is not None: + config['consumer_timeout_ms'] = consumer_timeout_ms + + consumer = KafkaConsumer(**config) + consumer.subscribe(['mv3dt']) + print("Connected to Kafka and subscribed to 'mv3dt' topic") + return consumer + +def draw_objects_on_map(frame_data, T_ov2px, map_img, trajectories, object_colors, frame_id, frame_history, show_ids=False, average_multi_cam=False): + vis_img = map_img.copy() + colors = [ + (255, 0, 0), (0, 255, 255), (139, 69, 19), (0, 255, 0), (255, 0, 255), + (50, 205, 50), (255, 140, 0), (0, 0, 255), (255, 165, 0), (255, 105, 180), + (75, 0, 130), (255, 255, 0), (0, 128, 128), (0, 191, 255), (154, 205, 50), + (255, 20, 147), (30, 144, 255), (128, 0, 128), (220, 20, 60), (0, 206, 209) + ] + + all_objects = [obj for frame_dict in frame_data.values() for obj in frame_dict.get('objects', [])] + + try: + current_frame_num = int(frame_id) + frame_history.append(current_frame_num) + if len(frame_history) > 240: + frame_history = frame_history[-240:] + except: + current_frame_num = 0 + + current_objects = set() + + if average_multi_cam: + # Group objects by ID across all cameras for averaging + objects_by_id = defaultdict(list) + for obj in all_objects: + bbox_3d = obj.get('bbox3d', {}).get('coordinates', {}) + if bbox_3d: + try: + world_x, world_y = bbox_3d[:2] + object_id = obj.get('id', 0) + objects_by_id[object_id].append((world_x, world_y)) + except: + continue + + # Calculate average positions and add to trajectories + for object_id, positions in objects_by_id.items(): + if positions: + # Calculate average world position + avg_world_x = sum(pos[0] for pos in positions) / len(positions) + avg_world_y = sum(pos[1] for pos in positions) / len(positions) + + try: + # Convert to pixel coordinates + pt_ov_h = np.array([avg_world_x, avg_world_y, 1.0]) + pt_px_h = np.dot(T_ov2px, pt_ov_h) + pt_px_h /= pt_px_h[2] + px_x, px_y = int(pt_px_h[0]), int(pt_px_h[1]) + + current_objects.add(object_id) + + if object_id not in object_colors: + object_colors[object_id] = colors[len(object_colors) % len(colors)] + + trajectories[object_id].append((px_x, px_y, current_frame_num)) + except: + continue + else: + # Original behavior: show all trajectory points from all cameras + for obj in all_objects: + bbox_3d = obj.get('bbox3d', {}).get('coordinates', {}) + if not bbox_3d: + continue + try: + world_x, world_y = bbox_3d[:2] + pt_ov_h = np.array([world_x, world_y, 1.0]) + pt_px_h = np.dot(T_ov2px, pt_ov_h) + pt_px_h /= pt_px_h[2] + px_x, px_y = int(pt_px_h[0]), int(pt_px_h[1]) + + object_id = obj.get('id', 0) + current_objects.add(object_id) + + if object_id not in object_colors: + object_colors[object_id] = colors[len(object_colors) % len(colors)] + + trajectories[object_id].append((px_x, px_y, current_frame_num)) + except: + continue + + # Cleanup old trajectory points + frame_threshold = current_frame_num - 240 + for object_id in list(trajectories.keys()): + trajectories[object_id] = [(x, y, f) for x, y, f in trajectories[object_id] if f >= frame_threshold] + if not trajectories[object_id]: + del trajectories[object_id] + object_colors.pop(object_id, None) + + # Draw trajectories + for object_id, traj_points in trajectories.items(): + if not traj_points: + continue + color = object_colors.get(object_id, (128, 128, 128)) + base_alpha = 0.9 if object_id in current_objects else 0.6 + min_alpha = 0.3 # Minimum brightness to prevent complete black + + for i, (x, y, _) in enumerate(traj_points): + # Slower fade: use square root for gentler curve + fade_ratio = (i / max(1, len(traj_points) - 1)) ** 0.5 + fade = min_alpha + (base_alpha - min_alpha) * fade_ratio + fade_color = tuple(int(c * fade) for c in color) + cv2.circle(vis_img, (x, y), 1, fade_color, -1) + + # Draw current positions with ID labels + if average_multi_cam: + # Draw averaged positions for each object + objects_by_id = defaultdict(list) + for obj in all_objects: + bbox_3d = obj.get('bbox3d', {}).get('coordinates', {}) + if bbox_3d: + try: + world_x, world_y = bbox_3d[:2] + object_id = obj.get('id', 0) + objects_by_id[object_id].append((world_x, world_y)) + except: + continue + + for object_id, positions in objects_by_id.items(): + if positions and object_id in object_colors: + # Calculate average world position + avg_world_x = sum(pos[0] for pos in positions) / len(positions) + avg_world_y = sum(pos[1] for pos in positions) / len(positions) + + try: + # Convert to pixel coordinates + pt_ov_h = np.array([avg_world_x, avg_world_y, 1.0]) + pt_px_h = np.dot(T_ov2px, pt_ov_h) + pt_px_h /= pt_px_h[2] + px_x, px_y = int(pt_px_h[0]), int(pt_px_h[1]) + + # Draw object circle + cv2.circle(vis_img, (px_x, px_y), 3, object_colors[object_id], -1) + + # Draw ID label near the object (if enabled) + if show_ids: + label_x = px_x + 8 + label_y = px_y - 8 + cv2.putText(vis_img, str(object_id), (label_x, label_y), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) + except: + continue + else: + # Original behavior: draw all object positions from all cameras + for obj in all_objects: + bbox_3d = obj.get('bbox3d', {}).get('coordinates', {}) + if bbox_3d: + try: + world_x, world_y = bbox_3d[:2] + pt_ov_h = np.array([world_x, world_y, 1.0]) + pt_px_h = np.dot(T_ov2px, pt_ov_h) + pt_px_h /= pt_px_h[2] + px_x, px_y = int(pt_px_h[0]), int(pt_px_h[1]) + + object_id = obj.get('id', 0) + if object_id in object_colors: + # Draw object circle + cv2.circle(vis_img, (px_x, px_y), 3, object_colors[object_id], -1) + + # Draw ID label near the object (if enabled) + if show_ids: + label_x = px_x + 8 + label_y = px_y - 8 + cv2.putText(vis_img, str(object_id), (label_x, label_y), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) + except: + continue + + return vis_img, frame_history + +def collect_all_messages(consumer, expected_sensors, max_timeout=300, verbose=False): + frame_buffer = FrameBuffer(expected_sensors=expected_sensors) + start_time = time.time() + last_msg_time = start_time + count = 0 + first_frame = True + + consumer._consumer_timeout_ms = 1000 + + while True: + current_time = time.time() + if current_time - start_time > max_timeout or current_time - last_msg_time > 10: + break + + batch = consumer.poll(timeout_ms=1000) + if not batch: + continue + + for _, messages in batch.items(): + for msg in messages: + try: + frame = Frame() + frame.ParseFromString(msg.value) + frame_dict = MessageToDict(frame) + frame_id = frame_dict.get('id', 'unknown') + + if first_frame and frame_id != 'unknown': + try: + if int(frame_id) > 100: + if verbose: + print(f"Discarding frame {frame_id} (from previous run)") + first_frame = False + continue + except (ValueError, TypeError): + pass + first_frame = False + + frame_buffer.add_frame(frame_id, frame_dict.get('sensorId', 'unknown'), frame_dict) + count += 1 + last_msg_time = current_time + except: + continue + + print(f"Collected {count} messages") + return frame_buffer + +def generate_video(dataset_path, output_path, show_ids, expected_sensors, average_multi_cam, verbose=False): + os.makedirs(output_path, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + video_output_path = os.path.join(output_path, f"trajectory_video_{timestamp}.mp4") + + # Setup common components + map_img, T_ov2px = load_map_and_transforms(dataset_path) + map_img_resized, scale_matrix, new_width, new_height = setup_map_scaling(map_img) + T_ov2px_scaled = np.dot(scale_matrix, T_ov2px) + + try: + consumer = create_kafka_consumer('mv3dt_bev_video') + + frame_buffer = collect_all_messages(consumer, expected_sensors, verbose=verbose) + complete_frames = frame_buffer.get_all_complete_frames() + print(frame_buffer.frame_data) + + if not complete_frames: + return + + fourcc = cv2.VideoWriter_fourcc(*'mp4v') + video_writer = cv2.VideoWriter(video_output_path, fourcc, 30, (new_width, new_height)) + + trajectories, object_colors, frame_history = defaultdict(list), {}, [] + + for frame_id, frame_data in tqdm(complete_frames): + vis_img, frame_history = draw_objects_on_map( + frame_data, T_ov2px_scaled, map_img_resized, trajectories, object_colors, frame_id, frame_history, show_ids, average_multi_cam) + + cv2.putText(vis_img, f"Frame: {frame_id}", + (10, vis_img.shape[0] - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) + video_writer.write(vis_img) + + video_writer.release() + print(f"Video saved: {video_output_path}") + + except Exception as e: + print(f"Error: {e}") + finally: + if 'consumer' in locals(): + consumer.close() + +def real_time_visualization(dataset_path, output_path, show_ids, expected_sensors, average_multi_cam, verbose=False): + # Setup common components + map_img, T_ov2px = load_map_and_transforms(dataset_path) + map_img_resized, scale_matrix, new_width, new_height = setup_map_scaling(map_img) + T_ov2px_scaled = np.dot(scale_matrix, T_ov2px) + + frame_buffer = FrameBuffer(expected_sensors=expected_sensors) + trajectories, object_colors, frame_history = defaultdict(list), {}, [] + + cv2.namedWindow('Bird-Eye View of Multi-View 3D Tracking', cv2.WINDOW_NORMAL) + cv2.resizeWindow('Bird-Eye View of Multi-View 3D Tracking', new_width, new_height) + + # Display blank map initially + initial_img = map_img_resized.copy() + cv2.imshow('Bird-Eye View of Multi-View 3D Tracking', initial_img) + cv2.waitKey(1) # Process window events + + consumer = None + video_writer = None + + try: + consumer = create_kafka_consumer('mv3dt_visualizer', consumer_timeout_ms=50) + + # Wait for partition assignment (up to 5 seconds) + assignment_timeout = 5.0 + start_time = time.time() + while not consumer.assignment() and (time.time() - start_time) < assignment_timeout: + consumer.poll(timeout_ms=100) # This triggers partition assignment + time.sleep(0.1) + + print(f"Consumer assignment: {consumer.assignment()}") + if not consumer.assignment(): + print("Warning: No partitions assigned. Topic might not exist or have no partitions.") + + # Try to get topic metadata for debugging + try: + metadata = consumer.list_consumer_group_offsets() + topics = consumer.topics() + print(f"Available topics: {topics}") + if 'mv3dt' in topics: + partitions = consumer.partitions_for_topic('mv3dt') + print(f"Partitions for 'mv3dt' topic: {partitions}") + else: + print("Topic 'mv3dt' not found!") + except Exception as debug_e: + print(f"Could not get topic metadata: {debug_e}") + + except Exception as e: + print(f"Kafka connection failed: {e}") + + print("Controls: 'q'-quit, 'c'-clear, 'r'-record") + + last_update = time.time() + last_frame_id = None + try: + while True: + current_time = time.time() + + if consumer: + try: + batch = consumer.poll(timeout_ms=10) + for _, messages in batch.items(): + for msg in messages: + try: + frame = Frame() + frame.ParseFromString(msg.value) + frame_dict = MessageToDict(frame) + # print ('Received frame', frame_dict.get('id', 'unknown'), 'from sensor', frame_dict.get('sensorId', 'unknown')) + frame_buffer.add_frame(frame_dict.get('id', 'unknown'), + frame_dict.get('sensorId', 'unknown'), frame_dict) + except: + continue + except: + pass + + while True: + frame_id, frame_data = frame_buffer.get_complete_frame() + if not frame_data: + break + + # If the first frame is > 100, it's likely from previous run. + if last_frame_id is None and frame_id > 100: + if verbose: + print(f"Discarding frame {frame_id} (from previous run)") + continue + + if last_frame_id is not None and frame_id - last_frame_id > 1: + # Process missing frames in between + for missing_frame_id in range(last_frame_id + 1, frame_id): + missing_id, missing_data = frame_buffer.get_frame(missing_frame_id) + if missing_data: + # Render frame with tracking data + vis_img, frame_history = draw_objects_on_map(missing_data, T_ov2px_scaled, + map_img_resized, trajectories, object_colors, missing_frame_id, frame_history, show_ids, average_multi_cam) + display_frame(vis_img, missing_frame_id, video_writer) + else: + # Render empty frame with correct frame_id + empty_vis_img = map_img_resized.copy() + display_frame(empty_vis_img, missing_frame_id, video_writer) + if last_frame_id is not None and 0 < last_frame_id - frame_id < 30: + if verbose: + print(f"Received late message from frame {frame_id}, last_frame_id was {last_frame_id}") + print(f"Discarding frame {frame_id} (late message)") + continue + last_frame_id = frame_id + vis_img, frame_history = draw_objects_on_map(frame_data, T_ov2px_scaled, + map_img_resized, trajectories, object_colors, frame_id, frame_history, show_ids, average_multi_cam) + + display_frame(vis_img, frame_id, video_writer) + + last_update = current_time + + key = cv2.waitKey(1) & 0xFF + try: + window_closed = cv2.getWindowProperty('Bird-Eye View of Multi-View 3D Tracking', cv2.WND_PROP_VISIBLE) < 1 + except cv2.error: + window_closed = True + if key == ord('q') or window_closed: + break + elif key == ord('c'): + trajectories.clear() + object_colors.clear() + frame_history.clear() + print("Cleared trajectories") + elif key == ord('r'): + if video_writer is None: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + live_video_path = os.path.join(output_path, f"live_trajectory_{timestamp}.mp4") + os.makedirs(output_path, exist_ok=True) + fourcc = cv2.VideoWriter_fourcc(*'mp4v') + video_writer = cv2.VideoWriter(live_video_path, fourcc, 30, (new_width, new_height)) + print(f"Started recording: {live_video_path}" if video_writer.isOpened() else "Recording failed") + else: + video_writer.release() + video_writer = None + print("Stopped recording") + + finally: + if video_writer: + video_writer.release() + if consumer: + consumer.close() + cv2.destroyAllWindows() + +def parse_args(): + parser = argparse.ArgumentParser(description='Kafka BEV Online Visualizer') + parser.add_argument('--dataset-path', type=str, + default="datasets/mtmc_4cam", + help='Path to dataset)') + parser.add_argument('--msgconv-config', type=str, + default='config_msgconv.txt', + help='Path to message converter config file (config_msgconv.txt)') + parser.add_argument('--output-path', type=str, + default='output_videos', + help='Output directory for videos') + parser.add_argument('--offline', action='store_true', + help='Run in offline mode (save a video from all messages instead of real-time visualization). \ + Please run this script after launching the MV3DT app.') + parser.add_argument('--show-ids', action='store_true', + help='Show object IDs near trajectory heads') + parser.add_argument('--average-multi-cam', action='store_true', + help='Average trajectory points from multiple cameras for the same object') + parser.add_argument('--verbose', action='store_true', + help='Print warnings and diagnostic messages') + + return parser.parse_args() + +def main(): + args = parse_args() + expected_sensors = extract_expected_sensors(args.msgconv_config) + print(f"Expected sensors: {expected_sensors}") + if args.offline: + generate_video(args.dataset_path, args.output_path, args.show_ids, expected_sensors, args.average_multi_cam, args.verbose) + else: + real_time_visualization(args.dataset_path, args.output_path, args.show_ids, expected_sensors, args.average_multi_cam, args.verbose) + + +if __name__ == "__main__": + main() diff --git a/deepstream-tracker-3d-multi-view/utils/kafka_client.py b/deepstream-tracker-3d-multi-view/utils/kafka_client.py new file mode 100644 index 0000000..d22dbf3 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/utils/kafka_client.py @@ -0,0 +1,70 @@ +"""Simple Kafka client for receiving and decoding MV3DT protobuf messages.""" + +import json +from kafka import KafkaConsumer +from kafka.errors import KafkaError +from google.protobuf.json_format import MessageToDict +from schema_pb2 import Frame +import argparse + +def main(): + """Main function to consume and print Kafka messages.""" + parser = argparse.ArgumentParser() + parser.add_argument('--topic', type=str, default='mv3dt', help='Kafka topic to subscribe to') + parser.add_argument('--broker', type=str, default='localhost:9092', help='Kafka broker server (e.g., localhost:9092)') + args = parser.parse_args() + + topic = args.topic + broker = args.broker + + print(f"Starting Kafka client reading from broker '{broker}' topic '{topic}'...") + + try: + # Create Kafka consumer + consumer = KafkaConsumer( + topic, + bootstrap_servers=broker, + auto_offset_reset='earliest', + value_deserializer=lambda x: x # Keep as bytes for protobuf + ) + + print("Connected to Kafka. Waiting for messages...") + + message_count = 0 + + # Consume messages + for msg in consumer: + try: + # Parse protobuf message + frame = Frame() + frame.ParseFromString(msg.value) + # print ('frame', frame) + # Convert to dictionary and then to JSON + frame_dict = MessageToDict(frame) + json_str = json.dumps(frame_dict, indent=2) + + message_count += 1 + print(f"\n--- Message {message_count} ---") + # print(f"Frame ID: {frame_id}") + # print(f"Sensor ID: {sensor_id}") + # print(f"Objects: {object_count}") + print(f"JSON Data:") + print(json_str) + print("-" * 50) + + except Exception as e: + print(f"Error parsing message: {e}") + continue + + except KafkaError as e: + print(f"Kafka error: {e}") + except KeyboardInterrupt: + print("\nStopped by user") + except Exception as e: + print(f"Unexpected error: {e}") + finally: + print("Consumer stopped") + + +if __name__ == "__main__": + main() diff --git a/deepstream-tracker-3d-multi-view/utils/schema_pb2.py b/deepstream-tracker-3d-multi-view/utils/schema_pb2.py new file mode 100644 index 0000000..f7d91d5 --- /dev/null +++ b/deepstream-tracker-3d-multi-view/utils/schema_pb2.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: schema.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cschema.proto\x12\x02nv\x1a\x1fgoogle/protobuf/timestamp.proto\"\xa5\x03\n\x05\x46rame\x12\x0f\n\x07version\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12-\n\ttimestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x10\n\x08sensorId\x18\x04 \x01(\t\x12\x1b\n\x07objects\x18\x05 \x03(\x0b\x32\n.nv.Object\x12\x1c\n\x03\x66ov\x18\x06 \x03(\x0b\x32\x0f.nv.TypeMetrics\x12\x1d\n\x04rois\x18\x07 \x03(\x0b\x32\x0f.nv.TypeMetrics\x12 \n\x10socialDistancing\x18\x08 \x01(\x0b\x32\x06.nv.SD\x12&\n\x0csegmentation\x18\t \x01(\x0b\x32\x10.nv.Segmentation\x12%\n\x0cinteractions\x18\n \x03(\x0b\x32\x0f.nv.Interaction\x12#\n\x0b\x63ongestions\x18\x0b \x03(\x0b\x32\x0e.nv.Congestion\x12!\n\x04info\x18\x0c \x03(\x0b\x32\x13.nv.Frame.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x93\x03\n\x06Object\x12\n\n\x02id\x18\x01 \x01(\t\x12\x16\n\x04\x62\x62ox\x18\x02 \x01(\x0b\x32\x08.nv.Bbox\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x12\n\nconfidence\x18\x04 \x01(\x02\x12\"\n\x04info\x18\x05 \x03(\x0b\x32\x14.nv.Object.InfoEntry\x12 \n\tembedding\x18\x06 \x01(\x0b\x32\r.nv.Embedding\x12\x16\n\x04pose\x18\x07 \x01(\x0b\x32\x08.nv.Pose\x12\x16\n\x04gaze\x18\x08 \x01(\x0b\x32\x08.nv.Gaze\x12$\n\x0blipActivity\x18\t \x01(\x0b\x32\x0f.nv.LipActivity\x12\r\n\x05speed\x18\n \x01(\x02\x12\x0b\n\x03\x64ir\x18\x0b \x03(\x02\x12\"\n\ncoordinate\x18\x0c \x01(\x0b\x32\x0e.nv.Coordinate\x12\x1e\n\x08location\x18\r \x01(\x0b\x32\x0c.nv.Location\x12\x1a\n\x06\x62\x62ox3d\x18\x0e \x01(\x0b\x32\n.nv.Bbox3d\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"-\n\nCoordinate\x12\t\n\x01x\x18\x01 \x01(\x01\x12\t\n\x01y\x18\x02 \x01(\x01\x12\t\n\x01z\x18\x03 \x01(\x01\"1\n\x08Location\x12\x0b\n\x03lat\x18\x01 \x01(\x01\x12\x0b\n\x03lon\x18\x02 \x01(\x01\x12\x0b\n\x03\x61lt\x18\x03 \x01(\x01\"\xca\x01\n\x04\x42\x62ox\x12\r\n\x05leftX\x18\x01 \x01(\x02\x12\x0c\n\x04topY\x18\x02 \x01(\x02\x12\x0e\n\x06rightX\x18\x03 \x01(\x02\x12\x0f\n\x07\x62ottomY\x18\x04 \x01(\x02\x12!\n\nembeddings\x18\x05 \x03(\x0b\x32\r.nv.Embedding\x12\x12\n\nconfidence\x18\x06 \x01(\x02\x12 \n\x04info\x18\x07 \x03(\x0b\x32\x12.nv.Bbox.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xa5\x01\n\x06\x42\x62ox3d\x12\x13\n\x0b\x63oordinates\x18\x01 \x03(\x01\x12!\n\nembeddings\x18\x02 \x03(\x0b\x32\r.nv.Embedding\x12\x12\n\nconfidence\x18\x03 \x01(\x02\x12\"\n\x04info\x18\x04 \x03(\x0b\x32\x14.nv.Bbox3d.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"s\n\x0cSegmentation\x12\x0c\n\x04mask\x18\x01 \x03(\x05\x12(\n\x04info\x18\x02 \x03(\x0b\x32\x1a.nv.Segmentation.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xc4\x01\n\x0bTypeMetrics\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\r\n\x05\x63ount\x18\x03 \x01(\x05\x12#\n\x0b\x63oordinates\x18\x04 \x03(\x0b\x32\x0e.nv.Coordinate\x12\x11\n\tobjectIds\x18\x05 \x03(\t\x12\'\n\x04info\x18\x06 \x03(\x0b\x32\x19.nv.TypeMetrics.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"&\n\x07\x43luster\x12\x1b\n\x06points\x18\x01 \x03(\x0b\x32\x0b.nv.Point2D\"\x1f\n\x07Point2D\x12\t\n\x01x\x18\x01 \x01(\x01\x12\t\n\x01y\x18\x02 \x01(\x01\"\xa0\x01\n\x02SD\x12\x11\n\tthreshold\x18\x01 \x01(\x01\x12\x1b\n\x13proximityDetections\x18\x02 \x01(\x05\x12\x1d\n\x08\x63lusters\x18\x03 \x03(\x0b\x32\x0b.nv.Cluster\x12\x1e\n\x04info\x18\x04 \x03(\x0b\x32\x10.nv.SD.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"K\n\x07Polygon\x12 \n\x0b\x63oordinates\x18\x01 \x03(\x0b\x32\x0b.nv.Point2D\x12\x1e\n\x05holes\x18\x02 \x03(\x0b\x32\x0f.nv.PolygonHole\"/\n\x0bPolygonHole\x12 \n\x0b\x63oordinates\x18\x01 \x03(\x0b\x32\x0b.nv.Point2D\"\xbc\x01\n\x0bInteraction\x12\n\n\x02id\x18\x01 \x01(\t\x12\x11\n\tobjectIds\x18\x02 \x03(\t\x12#\n\x0b\x63oordinates\x18\x03 \x03(\x0b\x32\x0e.nv.Coordinate\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x12\'\n\x04info\x18\x05 \x03(\x0b\x32\x19.nv.Interaction.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x90\x01\n\nCongestion\x12\n\n\x02id\x18\x01 \x01(\t\x12\x11\n\tobjectIds\x18\x02 \x03(\t\x12\x0e\n\x06\x61mount\x18\x03 \x01(\x02\x12&\n\x04info\x18\x04 \x03(\x0b\x32\x18.nv.Congestion.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9a\x02\n\x04Pose\x12\x0c\n\x04type\x18\x01 \x01(\t\x12$\n\tkeypoints\x18\x02 \x03(\x0b\x32\x11.nv.Pose.Keypoint\x12 \n\x07\x61\x63tions\x18\x03 \x03(\x0b\x32\x0f.nv.Pose.Action\x12 \n\x04info\x18\x04 \x03(\x0b\x32\x12.nv.Pose.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x41\n\x08Keypoint\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x63oordinates\x18\x02 \x03(\x02\x12\x12\n\nquaternion\x18\x03 \x03(\x02\x1a*\n\x06\x41\x63tion\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x12\n\nconfidence\x18\x02 \x01(\x02\"C\n\x04Gaze\x12\t\n\x01x\x18\x01 \x01(\x02\x12\t\n\x01y\x18\x02 \x01(\x02\x12\t\n\x01z\x18\x03 \x01(\x02\x12\r\n\x05theta\x18\x04 \x01(\x02\x12\x0b\n\x03phi\x18\x05 \x01(\x02\"!\n\x0bLipActivity\x12\x12\n\nclassLabel\x18\x01 \x01(\t\"q\n\x05\x45vent\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12!\n\x04info\x18\x05 \x03(\x0b\x32\x13.nv.Event.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xad\x01\n\x0f\x41nalyticsModule\x12\n\n\x02id\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0e\n\x06source\x18\x03 \x01(\t\x12\x0f\n\x07version\x18\x04 \x01(\t\x12+\n\x04info\x18\x05 \x03(\x0b\x32\x1d.nv.AnalyticsModule.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xcc\x01\n\x06Sensor\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x1e\n\x08location\x18\x04 \x01(\x0b\x32\x0c.nv.Location\x12\"\n\ncoordinate\x18\x05 \x01(\x0b\x32\x0e.nv.Coordinate\x12\"\n\x04info\x18\x06 \x03(\x0b\x32\x14.nv.Sensor.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xc3\x01\n\x05Place\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x1e\n\x08location\x18\x04 \x01(\x0b\x32\x0c.nv.Location\x12\"\n\ncoordinate\x18\x05 \x01(\x0b\x32\x0e.nv.Coordinate\x12!\n\x04info\x18\x06 \x03(\x0b\x32\x13.nv.Place.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8c\x02\n\x07Message\x12\x11\n\tmessageid\x18\x01 \x01(\t\x12\x12\n\nmdsversion\x18\x02 \x01(\t\x12-\n\ttimestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x18\n\x05place\x18\x04 \x01(\x0b\x32\t.nv.Place\x12\x1a\n\x06sensor\x18\x05 \x01(\x0b\x32\n.nv.Sensor\x12,\n\x0f\x61nalyticsModule\x18\x06 \x01(\x0b\x32\x13.nv.AnalyticsModule\x12\x1a\n\x06object\x18\x07 \x01(\x0b\x32\n.nv.Object\x12\x18\n\x05\x65vent\x18\x08 \x01(\x0b\x32\t.nv.Event\x12\x11\n\tvideoPath\x18\t \x01(\t\"s\n\tEmbedding\x12\x12\n\x06vector\x18\x01 \x03(\x02\x42\x02\x10\x01\x12%\n\x04info\x18\x02 \x03(\x0b\x32\x17.nv.Embedding.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xae\x01\n\tImageData\x12\x1f\n\x06\x66ormat\x18\x01 \x01(\x0e\x32\x0f.nv.ImageFormat\x12\x10\n\x08\x65ncoding\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c\x12%\n\x04info\x18\x05 \x03(\x0b\x32\x17.nv.ImageData.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xa4\x02\n\tVisionLLM\x12\x0f\n\x07version\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\'\n\x03\x65nd\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0cstartFrameId\x18\x04 \x01(\t\x12\x12\n\nendFrameId\x18\x05 \x01(\t\x12\x1a\n\x06sensor\x18\x06 \x01(\x0b\x32\n.nv.Sensor\x12\x14\n\x03llm\x18\x07 \x01(\x0b\x32\x07.nv.LLM\x12%\n\x04info\x18\x08 \x03(\x0b\x32\x17.nv.VisionLLM.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9f\x01\n\x03LLM\x12\x1f\n\x04info\x18\x01 \x03(\x0b\x32\x11.nv.LLM.InfoEntry\x12\'\n\rconversations\x18\x02 \x03(\x0b\x32\x10.nv.Conversation\x12!\n\nembeddings\x18\x03 \x03(\x0b\x32\r.nv.Embedding\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\";\n\x0c\x43onversation\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x10\n\x08response\x18\x03 \x01(\t*2\n\x0bImageFormat\x12\x07\n\x03RAW\x10\x00\x12\x07\n\x03JPG\x10\x01\x12\x08\n\x04JPEG\x10\x02\x12\x07\n\x03PNG\x10\x03\x62\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'schema_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _FRAME_INFOENTRY._options = None + _FRAME_INFOENTRY._serialized_options = b'8\001' + _OBJECT_INFOENTRY._options = None + _OBJECT_INFOENTRY._serialized_options = b'8\001' + _BBOX_INFOENTRY._options = None + _BBOX_INFOENTRY._serialized_options = b'8\001' + _BBOX3D_INFOENTRY._options = None + _BBOX3D_INFOENTRY._serialized_options = b'8\001' + _SEGMENTATION_INFOENTRY._options = None + _SEGMENTATION_INFOENTRY._serialized_options = b'8\001' + _TYPEMETRICS_INFOENTRY._options = None + _TYPEMETRICS_INFOENTRY._serialized_options = b'8\001' + _SD_INFOENTRY._options = None + _SD_INFOENTRY._serialized_options = b'8\001' + _INTERACTION_INFOENTRY._options = None + _INTERACTION_INFOENTRY._serialized_options = b'8\001' + _CONGESTION_INFOENTRY._options = None + _CONGESTION_INFOENTRY._serialized_options = b'8\001' + _POSE_INFOENTRY._options = None + _POSE_INFOENTRY._serialized_options = b'8\001' + _EVENT_INFOENTRY._options = None + _EVENT_INFOENTRY._serialized_options = b'8\001' + _ANALYTICSMODULE_INFOENTRY._options = None + _ANALYTICSMODULE_INFOENTRY._serialized_options = b'8\001' + _SENSOR_INFOENTRY._options = None + _SENSOR_INFOENTRY._serialized_options = b'8\001' + _PLACE_INFOENTRY._options = None + _PLACE_INFOENTRY._serialized_options = b'8\001' + _EMBEDDING_INFOENTRY._options = None + _EMBEDDING_INFOENTRY._serialized_options = b'8\001' + _EMBEDDING.fields_by_name['vector']._options = None + _EMBEDDING.fields_by_name['vector']._serialized_options = b'\020\001' + _IMAGEDATA_INFOENTRY._options = None + _IMAGEDATA_INFOENTRY._serialized_options = b'8\001' + _VISIONLLM_INFOENTRY._options = None + _VISIONLLM_INFOENTRY._serialized_options = b'8\001' + _LLM_INFOENTRY._options = None + _LLM_INFOENTRY._serialized_options = b'8\001' + _IMAGEFORMAT._serialized_start=4538 + _IMAGEFORMAT._serialized_end=4588 + _FRAME._serialized_start=54 + _FRAME._serialized_end=475 + _FRAME_INFOENTRY._serialized_start=432 + _FRAME_INFOENTRY._serialized_end=475 + _OBJECT._serialized_start=478 + _OBJECT._serialized_end=881 + _OBJECT_INFOENTRY._serialized_start=432 + _OBJECT_INFOENTRY._serialized_end=475 + _COORDINATE._serialized_start=883 + _COORDINATE._serialized_end=928 + _LOCATION._serialized_start=930 + _LOCATION._serialized_end=979 + _BBOX._serialized_start=982 + _BBOX._serialized_end=1184 + _BBOX_INFOENTRY._serialized_start=432 + _BBOX_INFOENTRY._serialized_end=475 + _BBOX3D._serialized_start=1187 + _BBOX3D._serialized_end=1352 + _BBOX3D_INFOENTRY._serialized_start=432 + _BBOX3D_INFOENTRY._serialized_end=475 + _SEGMENTATION._serialized_start=1354 + _SEGMENTATION._serialized_end=1469 + _SEGMENTATION_INFOENTRY._serialized_start=432 + _SEGMENTATION_INFOENTRY._serialized_end=475 + _TYPEMETRICS._serialized_start=1472 + _TYPEMETRICS._serialized_end=1668 + _TYPEMETRICS_INFOENTRY._serialized_start=432 + _TYPEMETRICS_INFOENTRY._serialized_end=475 + _CLUSTER._serialized_start=1670 + _CLUSTER._serialized_end=1708 + _POINT2D._serialized_start=1710 + _POINT2D._serialized_end=1741 + _SD._serialized_start=1744 + _SD._serialized_end=1904 + _SD_INFOENTRY._serialized_start=432 + _SD_INFOENTRY._serialized_end=475 + _POLYGON._serialized_start=1906 + _POLYGON._serialized_end=1981 + _POLYGONHOLE._serialized_start=1983 + _POLYGONHOLE._serialized_end=2030 + _INTERACTION._serialized_start=2033 + _INTERACTION._serialized_end=2221 + _INTERACTION_INFOENTRY._serialized_start=432 + _INTERACTION_INFOENTRY._serialized_end=475 + _CONGESTION._serialized_start=2224 + _CONGESTION._serialized_end=2368 + _CONGESTION_INFOENTRY._serialized_start=432 + _CONGESTION_INFOENTRY._serialized_end=475 + _POSE._serialized_start=2371 + _POSE._serialized_end=2653 + _POSE_INFOENTRY._serialized_start=432 + _POSE_INFOENTRY._serialized_end=475 + _POSE_KEYPOINT._serialized_start=2544 + _POSE_KEYPOINT._serialized_end=2609 + _POSE_ACTION._serialized_start=2611 + _POSE_ACTION._serialized_end=2653 + _GAZE._serialized_start=2655 + _GAZE._serialized_end=2722 + _LIPACTIVITY._serialized_start=2724 + _LIPACTIVITY._serialized_end=2757 + _EVENT._serialized_start=2759 + _EVENT._serialized_end=2872 + _EVENT_INFOENTRY._serialized_start=432 + _EVENT_INFOENTRY._serialized_end=475 + _ANALYTICSMODULE._serialized_start=2875 + _ANALYTICSMODULE._serialized_end=3048 + _ANALYTICSMODULE_INFOENTRY._serialized_start=432 + _ANALYTICSMODULE_INFOENTRY._serialized_end=475 + _SENSOR._serialized_start=3051 + _SENSOR._serialized_end=3255 + _SENSOR_INFOENTRY._serialized_start=432 + _SENSOR_INFOENTRY._serialized_end=475 + _PLACE._serialized_start=3258 + _PLACE._serialized_end=3453 + _PLACE_INFOENTRY._serialized_start=432 + _PLACE_INFOENTRY._serialized_end=475 + _MESSAGE._serialized_start=3456 + _MESSAGE._serialized_end=3724 + _EMBEDDING._serialized_start=3726 + _EMBEDDING._serialized_end=3841 + _EMBEDDING_INFOENTRY._serialized_start=432 + _EMBEDDING_INFOENTRY._serialized_end=475 + _IMAGEDATA._serialized_start=3844 + _IMAGEDATA._serialized_end=4018 + _IMAGEDATA_INFOENTRY._serialized_start=432 + _IMAGEDATA_INFOENTRY._serialized_end=475 + _VISIONLLM._serialized_start=4021 + _VISIONLLM._serialized_end=4313 + _VISIONLLM_INFOENTRY._serialized_start=432 + _VISIONLLM_INFOENTRY._serialized_end=475 + _LLM._serialized_start=4316 + _LLM._serialized_end=4475 + _LLM_INFOENTRY._serialized_start=432 + _LLM_INFOENTRY._serialized_end=475 + _CONVERSATION._serialized_start=4477 + _CONVERSATION._serialized_end=4536 +# @@protoc_insertion_point(module_scope) diff --git a/deepstream-tracker-3d/README.md b/deepstream-tracker-3d/README.md new file mode 100644 index 0000000..c4ce75e --- /dev/null +++ b/deepstream-tracker-3d/README.md @@ -0,0 +1,121 @@ +# Single-View 3D Tracking in DeepStream + +## Introduction +This sample application demonstrates the single-view 3D tracking with DeepStream SDK. Given the [camera matrix and human model](configs/camInfo.yml) of a static camera, this application estimates and keeps tracking of object states in the 3D physical world. It can recover the complete bounding box, foot location and body convex hulls precisely from partial occlusions. For algorithm and setup details, please refer to [DeepStream Single View 3D Tracking Documentation](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvtracker.html#single-view-3d-tracking-alpha). + +The currnet SV3DT configuration in this sample uses an 2D pose estimator, which estimates human key-points on the 2D image plane. The SV3DT algorithm uses height and waist key-points as anchor to precisely estimate each person's 3D human height. If setting `poseEstimatorType: 0` in `configs/config_tracker_NvDCF_accuracy_3D.yml`, pose estimator will be disabled, and the algorithm uses 2D detection bounding boxes and a human model with fixed height. It estimates the 3D location by matching the head with 2D bounding box's top edge. + +## Prerequisites +This sample application can be run on both x86 and Jetson platforms inside DeepStream container. Check [here](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_docker_containers.html#prerequisites) for DeepStream container setup. +1. Download the latest DeepStream container image from NGC (e.g., DS 9.0 in the example below) + ```bash + export DS_IMG_NAME="nvcr.io/nvidia/deepstream:9.0-triton-multiarch" + docker pull $DS_IMG_NAME + ``` + +2. Git clone the current `deepstream_reference_apps` repository to the host machine, and enter single-view 3D tracking directory inside the repository. + ```bash + git clone https://github.com/NVIDIA-AI-IOT/deepstream_reference_apps.git + cd deepstream_reference_apps/deepstream-tracker-3d + ``` + +3. Download NVIDIA pretrained [`PeopleNet`](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet/files?version=deployable_quantized_onnx_v2.6.3) for 2D detection. + + ```bash + # current directory: deepstream_reference_apps/deepstream-tracker-3d + mkdir -p models/PeopleNet + cd models/PeopleNet; wget --no-check-certificate --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet/versions/deployable_quantized_onnx_v2.6.3/zip -O peoplenet_deployable_quantized_onnx_v2.6.3.zip; unzip peoplenet_deployable_quantized_onnx_v2.6.3.zip + ``` + + The model files are now stored in `PeopleNet` directory as + + ```bash + deepstream-tracker-3d + ├── configs + ├── streams + └── models + └── PeopleNet + ├── labels.txt + ├── resnet34_peoplenet.onnx + └── resnet34_peoplenet_int8.txt + ``` +## Running the Application +Launch the container from current directory, and execute the 3D tracking pipeline inside the container. The current [config](configs/deepstream_app_source1_3d_tracking.txt) requires users to run with a display because it uses EGL sink to visualize the overlay results. To run through ssh without display, please change `type=2` to `1` in group `[sink0]` in that file. Users can check [DeepStream sink group](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_ref_app_deepstream.html#sink-group) for the usage of each sink. + +```bash +cd ../.. +sudo xhost + # give container access to display +# current directory: deepstream_reference_apps/deepstream-tracker-3d +docker run --runtime=nvidia -it --rm --net=host --privileged -v /tmp/.X11-unix:/tmp/.X11-unix -v $(pwd):/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-tracker-3d -e DISPLAY=$DISPLAY $DS_IMG_NAME +``` + +Inside container, run the following commands. Please note that when `deepstream-app` is launched for the first time, it tries to create model engine files, which may take a couple minutes, depending on HW platforms. + +```bash +# Install prerequisites +cd /opt/nvidia/deepstream/deepstream/ +bash user_additional_install.sh + +# Download ReID and BodyPose3DNet model +export MODEL_DIR="/opt/nvidia/deepstream/deepstream/samples/models/Tracker" +mkdir -p $MODEL_DIR +wget 'https://api.ngc.nvidia.com/v2/models/nvidia/tao/reidentificationnet/versions/deployable_v1.0/files/resnet50_market1501.etlt' -P $MODEL_DIR +wget 'https://api.ngc.nvidia.com/v2/models/nvidia/tao/bodypose3dnet/versions/deployable_accuracy_onnx_1.0/files/bodypose3dnet_accuracy.onnx' -P $MODEL_DIR + +# Run 3D tracking pipeline +cd /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-tracker-3d/configs +mkdir -p track_results +deepstream-app -c deepstream_app_source1_3d_tracking.txt +``` + +## Output Retrival and Visualization + +### DeepStream Direct Visualization +When the pipeline is launced, DeepStream shows the output video like below while processing the input video. The 3D bounding boxes of the people are reconstructed and plotted even though there are partial occlusions. The result video is saved as `out.mp4`. + +![sample 3d tracking results](figures/.retail_osd.png) + +### 3D Metadata Processing and Visualization +The extracted metadata (e.g., bounding box, frame num, target ID, etc.) is saved in both extended MOT and KITTI format. For detailed explanation, please refer to [DeepStream Tracker Miscellaneous Data Output](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvtracker.html#miscellaneous-data-output). + +The MOT results can be found in `track_dump_0.txt` file, which contains all the object metadata for one stream, and the data format is defined below. The foot image position and the convex hull of the projected cylindrical human model are defined in video frame coordinates, and can be used to draw the visualization figures below. Users can create such overlay video or image using their favorite tools like [OpenCV](https://github.com/opencv/opencv). The foot world position is defined in the 3D world ground plane corresponding to the 3x4 camera projection matrix. + +| frame number(starting from 1) | object unique id | bbox left | bbox top | bbox width | bbox height | confidence | Foot World Position X | Foot World Position Y | blank | class id | tracker state | visibility | Foot Image Position X | Foot Image Position Y | ConvexHull Points Relative to bbox center | +|-------------------------------|------------------|-----------|----------|------------|-------------|------------|-----------------------|-----------------------|-------|--------------|---------------|------------|-----------------------|-----------------------|--------------------------------| +| unsigned int |long unsigned int | int | int | int | int | float | float | float | int | unsigned int | int | float | float | float | int separated by vertical bar | + +Sample output is like below. The green dot in the visualization figure is plotted as `(Foot Image Position X, Foot Image Position Y)`, and the cylinder is plotted by connecting convex hull points. For example, the bbox in the first line of the output file has center `(1433, -50)`. Then its convex hull points are `(1433 - 94, -50 - 170), (1433 - 87, -50 - 176), ..., (1433 - 23, -50 + 190)`. Users can plot the foot location and convex hull on their own as shown in the figure below. +```txt +1,1,1366,-195,134,290,1.000,-171.080,1058.295,-1,0,2,0.220,1458,80,-94|-170|-87|-176|-71|-183|-49|-191|-23|-196|0|-200|18|-201|29|-198|95|165|95|173|85|183|66|191|42|198|16|202|-4|201|-18|197|-23|190 +2,1,1365,-194,135,290,0.989,-170.646,1045.254,-1,0,2,0.230,1458,84,-94|-170|-87|-176|-71|-183|-49|-191|-23|-196|0|-200|18|-201|29|-198|95|165|95|173|85|183|66|191|42|198|16|202|-4|201|-18|197|-23|190 +3,1,1366,-196,134,290,0.860,-170.679,1054.089,-1,0,2,0.229,1458,82,-94|-170|-87|-176|-71|-183|-49|-191|-23|-196|0|-200|18|-201|29|-198|95|165|95|173|85|183|66|191|42|198|16|202|-4|201|-18|197|-23|190 +... +``` +![sample 3d tracking results](figures/.retail_viz.png) + +The KITTI results can be found in `track_results` folder. A file will be created for each frame in each stream, and the data format is defined below. + +| object Label | object Unique Id | blank | blank | blank | bbox left | bbox top | bbox right | bbox bottom | blank | blank | blank | blank | blank | blank | blank |confidence | visibility (optional) | Foot Image Position X (optional) | Foot Image Position Y (optional) | +|--------------|------------------|-------|-------|-------|-----------|----------|------------|-------------|-------|-------|-------|-------|-------|-------|-------|-----------|-----------------------|-----------------------|-----------------------| +| string | long unsigned | float | int | float | float | float | float | float | float | float | float | float | float | float | float | float | float | float | float | + +Each frame is saved as `track_results/00_000_xxxxxx.txt`. Sample output of a frame is like below. Note that if a object is found in past frame data, it wouldn't have visibility and foot position in KITTI dump. +```txt +person 1 0.0 0 0.0 1365.907227 -196.875290 1500.249146 93.554581 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.890186 0.229870 1458.166992 81.585060 +person 6 0.0 0 0.0 1419.655151 72.774818 1647.446167 561.540894 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.841420 0.408513 1575.264160 531.851746 +person 0 0.0 0 0.0 1008.387817 36.228714 1202.886353 421.609314 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.632065 0.504738 1148.117188 399.992645 +... +``` + +## Customizing the Video +To run the single view 3D tracking on other videos, the following changes are required. +1. In `deepstream_app_source1_3d_tracking.txt`, change `uri=file://../streams/Retail02_short.mp4` to the new video name, `width=1920, height=1080, tracker-width=1920, tracker-height=1080` to the new video's resolution. +2. Generate camera projection matrix for the new video. Change `projectionMatrix_3x4` in `camInfo.yml` into the new matrix. + +Note: Multiple streams can run at the same time. Add all the sources to `deepstream_app_source1_3d_tracking.txt`, and all the camera information files to `config_tracker_NvDCF_accuracy_3D.yml`. +```yaml + cameraModelFilepath: # In order of the source streams + - 'camInfo-01.yml' + - 'camInfo-02.yml' + - ... +``` diff --git a/deepstream-tracker-3d/configs/camInfo.yml b/deepstream-tracker-3d/configs/camInfo.yml new file mode 100644 index 0000000..1e4c41f --- /dev/null +++ b/deepstream-tracker-3d/configs/camInfo.yml @@ -0,0 +1,38 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# row-major, so the actual matrix is like +# 2582.5691623002185 -485.10283397043617 650.27745033162591 89466.605755471101 +# -423.46809686390498 1044.6870098337931 2461.1283636622838 -214284.36100320917 +# -0.25563255317172684 -0.90495941862094287 0.34014768617197644 -1181.960782357068 + + +projectionMatrix_3x4: + - 2582.5691623002185 + - -485.10283397043617 + - 650.27745033162591 + - -89466.605755471101 + - -423.46809686390498 + - 1044.6870098337931 + - 2461.1283636622838 + - -214284.36100320917 + - -0.25563255317172684 + - -0.90495941862094287 + - 0.34014768617197644 + - -1181.960782357068 + +# the height and radius of the cylinder model +modelInfo: + height: 205 + radius: 33 diff --git a/deepstream-tracker-3d/configs/config_infer_primary.txt b/deepstream-tracker-3d/configs/config_infer_primary.txt new file mode 100644 index 0000000..0ed48eb --- /dev/null +++ b/deepstream-tracker-3d/configs/config_infer_primary.txt @@ -0,0 +1,47 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 + +infer-dims=3;544;960 +int8-calib-file=../models/PeopleNet/resnet34_peoplenet_int8.txt +model-engine-file=../models/PeopleNet/resnet34_peoplenet.onnx_b1_gpu0_fp16.engine +labelfile-path=../models/PeopleNet/labels.txt +onnx-file=../models/PeopleNet/resnet34_peoplenet.onnx + +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=3 +interval=0 +gie-unique-id=1 +## 0=Group Rectangles, 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +cluster-mode=3 +#enable-dla=1 +#use-dla-core=0 +#scaling-filter=4 + +filter-out-class-ids=1;2 + +[class-attrs-all] +pre-cluster-threshold=0.1429 +nms-iou-threshold=0.4688 +minBoxes=3 +dbscan-min-score=0.7726 +eps=0.2538 +detected-min-w=20 +detected-min-h=20 diff --git a/deepstream-tracker-3d/configs/config_tracker_NvDCF_accuracy_3D.yml b/deepstream-tracker-3d/configs/config_tracker_NvDCF_accuracy_3D.yml new file mode 100644 index 0000000..461a14c --- /dev/null +++ b/deepstream-tracker-3d/configs/config_tracker_NvDCF_accuracy_3D.yml @@ -0,0 +1,158 @@ +%YAML:1.0 +################################################################################ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +BaseConfig: + minDetectorConfidence: 0.1894 # If the confidence of a detector bbox is lower than this, then it won't be considered for tracking + +TargetManagement: + enableBboxUnClipping: 1 # In case the bbox is likely to be clipped by image border, unclip bbox + preserveStreamUpdateOrder: 0 # When assigning new target ids, preserve input streams' order to keep target ids in a deterministic order over multuple runs + maxTargetsPerStream: 150 # Max number of targets to track per stream. Recommended to set >10. Note: this value should account for the targets being tracked in shadow mode as well. Max value depends on the GPU memory capacity + + # [Creation & Termination Policy] + minIouDiff4NewTarget: 0.3686 # If the IOU between the newly detected object and any of the existing targets is higher than this threshold, this newly detected object will be discarded. + minTrackerConfidence: 0.1513 # If the confidence of an object tracker is lower than this on the fly, then it will be tracked in shadow mode. Valid Range: [0.0, 1.0] + probationAge: 2 # If the target's age exceeds this, the target will be considered to be valid. + maxShadowTrackingAge: 42 # Max length of shadow tracking. If the shadowTrackingAge exceeds this limit, the tracker will be terminated. + earlyTerminationAge: 1 # If the shadowTrackingAge reaches this threshold while in TENTATIVE period, the target will be terminated prematurely. + + # dump tracklets in txt file + outputTerminatedTracks: 1 # save terminated tracklets + terminatedTrackFilename: track_dump_ # file name: "terminatedTrackFilename"0.txt, "terminatedTrackFilename"1_2.txt, ... + +TrajectoryManagement: + useUniqueID: 0 # Use 64-bit long Unique ID when assignining tracker ID. Default is [true] + enableReAssoc: 1 # Enable Re-Assoc + + # [Re-Assoc Metric: Thresholds for valid candidates] + minMatchingScore4Overall: 0.6622 # min matching score for overall + minTrackletMatchingScore: 0.2940 # min tracklet similarity score for re-assoc + minMatchingScore4ReidSimilarity: 0.0771 # min reid similarity score for re-assoc + + # [Re-Assoc Metric: Weights] + matchingScoreWeight4TrackletSimilarity: 0.7981 # weight for tracklet similarity score + matchingScoreWeight4ReidSimilarity: 0.3848 # weight for reid similarity score + + # [Re-Assoc: Motion-based] + minTrajectoryLength4Projection: 34 # min trajectory length required to make projected trajectory + prepLength4TrajectoryProjection: 58 # the length of the trajectory during which the state estimator is updated to make projections + trajectoryProjectionLength: 33 # the length of the projected trajectory + maxAngle4TrackletMatching: 67 # max angle difference for tracklet matching [degree] + minSpeedSimilarity4TrackletMatching: 0.0574 # min speed similarity for tracklet matching + minBboxSizeSimilarity4TrackletMatching: 0.1013 # min bbox size similarity for tracklet matching + maxTrackletMatchingTimeSearchRange: 27 # the search space in time for max tracklet similarity + trajectoryProjectionProcessNoiseScale: 0.0100 # trajectory projector's process noise scale w.r.t. state estimator + trajectoryProjectionMeasurementNoiseScale: 100 # trajectory projector's measurement noise scale w.r.t. state estimator + trackletSpacialSearchRegionScale: 0.0100 # the search region scale for peer tracklet + + # [Re-Assoc: Reid based. Reid model params are set in ReID section] + reidExtractionInterval: 8 # frame interval to extract reid features per target + +DataAssociator: + dataAssociatorType: 0 # the type of data associator among { DEFAULT= 0 } + associationMatcherType: 1 # the type of matching algorithm among { GREEDY=0, CASCADED=1 } + checkClassMatch: 1 # If checked, only the same-class objects are associated with each other. Default: true + + # [Association Metric: Thresholds for valid candidates] + minMatchingScore4Overall: 0.0222 # Min total score + minMatchingScore4SizeSimilarity: 0.3552 # Min bbox size similarity score + minMatchingScore4Iou: 0.0548 # Min IOU score + minMatchingScore4VisualSimilarity: 0.5043 # Min visual similarity score + + # [Association Metric: Weights] + matchingScoreWeight4VisualSimilarity: 0.3951 # Weight for the visual similarity (in terms of correlation response ratio) + matchingScoreWeight4SizeSimilarity: 0.6003 # Weight for the Size-similarity score + matchingScoreWeight4Iou: 0.4033 # Weight for the IOU score + + # [Association Metric: Tentative detections] only uses iou similarity for tentative detections + tentativeDetectorConfidence: 0.1024 # If a detection's confidence is lower than this but higher than minDetectorConfidence, then it's considered as a tentative detection + minMatchingScore4TentativeIou: 0.2852 # Min iou threshold to match targets and tentative detection + +StateEstimator: + stateEstimatorType: 3 # the type of state estimator among { DUMMY=0, SIMPLE=1, REGULAR=2, 3D=3 } + + # [Dynamics Modeling] + processNoiseVar4Loc: 6810.8668 # Process noise variance for bbox center + processNoiseVar4Vel: 1348.4874 # Process noise variance for velocity + measurementNoiseVar4Detector: 100.0000 # Measurement noise variance for detector's detection + measurementNoiseVar4Tracker: 293.3238 # Measurement noise variance for tracker's localization + +ObjectModelProjection: + minPoseConfidence: 0.925 + cameraModelFilepath: # camera calibration file for each stream + - /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-tracker-3d/configs/camInfo.yml + outputVisibility: 1 # output visibility by occlusion + outputFootLocation: 1 # output foot location estimated from 3D model + outputConvexHull: 0 # output convex hull for each object estimated from 3D cylinder model + +VisualTracker: + visualTrackerType: 2 # the type of visual tracker among { DUMMY=0, NvDCF=1, NvDCF_VPI=2 } + vpiBackend4DcfTracker: 1 # the type of compute backend among {CUDA=1, PVA=2} + + # [NvDCF: Feature Extraction] + useColorNames: 1 # Use ColorNames feature + useHog: 1 # Use Histogram-of-Oriented-Gradient (HOG) feature + featureImgSizeLevel: 3 # Size of a feature image. Valid range: {1, 2, 3, 4, 5}, from the smallest to the largest + featureFocusOffsetFactor_y: -0.1054 # The offset for the center of hanning window relative to the feature height. The center of hanning window would move by (featureFocusOffsetFactor_y*featureMatSize.height) in vertical direction + + # [NvDCF: Correlation Filter] + filterLr: 0.0767 # learning rate for DCF filter in exponential moving average. Valid Range: [0.0, 1.0] + filterChannelWeightsLr: 0.0339 # learning rate for the channel weights among feature channels. Valid Range: [0.0, 1.0] + gaussianSigma: 0.5687 # Standard deviation for Gaussian for desired response when creating DCF filter [pixels] + +ReID: + reidType: 2 # The type of reid among { DUMMY=0, NvDEEPSORT=1, Reid based reassoc=2, both NvDEEPSORT and reid based reassoc=3} + + # [Reid Network Info] + batchSize: 100 # Batch size of reid network + workspaceSize: 1000 # Workspace size to be used by reid engine, in MB + reidFeatureSize: 256 # Size of reid feature + reidHistorySize: 100 # Max number of reid features kept for one object + inferDims: [3, 256, 128] # Reid network input dimension CHW or HWC based on inputOrder + networkMode: 1 # Reid network inference precision mode among {fp32=0, fp16=1, int8=2 } + + # [Input Preprocessing] + inputOrder: 0 # Reid network input order among { NCHW=0, NHWC=1 }. Batch will be converted to the specified order before reid input. + colorFormat: 0 # Reid network input color format among {RGB=0, BGR=1 }. Batch will be converted to the specified color before reid input. + offsets: [123.6750, 116.2800, 103.5300] # Array of values to be subtracted from each input channel, with length equal to number of channels + netScaleFactor: 0.01735207 # Scaling factor for reid network input after substracting offsets + keepAspc: 1 # Whether to keep aspc ratio when resizing input objects for reid + useVPICropScaler: 1 # Use VPI backend crop and scaler + + # [Output Postprocessing] + addFeatureNormalization: 1 # If reid feature is not normalized in network, adding normalization on output so each reid feature has l2 norm equal to 1 + minVisibility4GalleryUpdate: 0.6 # Add ReID embedding to the gallery only if the visibility is not lower than this + + # [Paths and Names] + tltEncodedModel: "/opt/nvidia/deepstream/deepstream/samples/models/Tracker/resnet50_market1501.etlt" # NVIDIA TAO model path + tltModelKey: "nvidia_tao" # NVIDIA TAO model key + modelEngineFile: "/opt/nvidia/deepstream/deepstream/samples/models/Tracker/resnet50_market1501.etlt_b100_gpu0_fp16.engine" # Engine file path + +PoseEstimator: + poseEstimatorType: 1 # Type of pose estimator used + useVPICropScaler: 1 # Use VPI backend for cropping and scaling + batchSize: 1 # Batch size for pose estimation + workspaceSize: 1000 # Workspace size in MB for the pose estimator engine + inferDims: [3, 256, 192] # Input dimensions for the pose estimator network (C, H, W) + networkMode: 1 # Inference precision mode (fp32=0, fp16=1, int8=2) + inputOrder: 0 # Input order for the network (NCHW=0, NHWC=1) + colorFormat: 0 # Input color format (RGB=0, BGR=1) + offsets: [123.6750, 116.2800, 103.5300] # Channel-wise mean subtraction values + netScaleFactor: 0.00392156 # Scaling factor for input normalization + onnxFile: "/opt/nvidia/deepstream/deepstream/samples/models/Tracker/bodypose3dnet_accuracy.onnx" # Path to the ONNX model file + modelEngineFile: "/opt/nvidia/deepstream/deepstream/samples/models/Tracker/bodypose3dnet_accuracy.onnx_b1_gpu0_fp16.engine" # Path to the engine file + poseInferenceInterval: -1 # Pose inference frame interval. -1 means only for the first frame of each target and use it to determine the target height. diff --git a/deepstream-tracker-3d/configs/deepstream_app_source1_3d_tracking.txt b/deepstream-tracker-3d/configs/deepstream_app_source1_3d_tracking.txt new file mode 100644 index 0000000..84ef117 --- /dev/null +++ b/deepstream-tracker-3d/configs/deepstream_app_source1_3d_tracking.txt @@ -0,0 +1,107 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[application] +enable-perf-measurement=1 +perf-measurement-interval-sec=3 +kitti-track-output-dir=track_results + +[tiled-display] +enable=1 +rows=1 +columns=1 +width=1280 +height=720 +gpu-id=0 +nvbuf-memory-type=0 + +[source0] +enable=1 +type=3 +uri=file://../streams/Retail02_short.mp4 +num-sources=1 +gpu-id=0 +cudadec-memtype=0 + +[sink0] +enable=1 +type=2 +sync=1 +source-id=0 +gpu-id=0 +nvbuf-memory-type=0 +qos=0 + +[sink1] +enable=1 +type=3 +container=1 +codec=1 +enc-type=0 +sync=0 +bitrate=2000000 +profile=0 +output-file=out.mp4 +source-id=0 + +[osd] +enable=1 +gpu-id=0 +border-width=2 +text-size=15 +text-color=1;1;1;1; +text-bg-color=0.3;0.3;0.3;1 +font=Serif +show-clock=0 +clock-x-offset=800 +clock-y-offset=820 +clock-text-size=12 +clock-color=1;0;0;0 +nvbuf-memory-type=0 + +[streammux] +gpu-id=0 +live-source=0 +batch-size=1 +batched-push-timeout=-1 +width=1920 +height=1080 +enable-padding=0 +nvbuf-memory-type=0 + +[primary-gie] +enable=1 +gpu-id=0 +batch-size=1 +bbox-border-color0=1;0;0;1 +bbox-border-color1=0;1;1;1 +bbox-border-color2=0;0;1;1 +bbox-border-color3=0;1;0;1 +gie-unique-id=1 +nvbuf-memory-type=0 +interval=0 +config-file=config_infer_primary.txt + +[tracker] +enable=1 +tracker-width=1920 +tracker-height=1080 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +ll-config-file=config_tracker_NvDCF_accuracy_3D.yml +#ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml +gpu-id=0 + +[tests] +file-loop=0 + diff --git a/deepstream-tracker-3d/figures/.retail_osd.png b/deepstream-tracker-3d/figures/.retail_osd.png new file mode 100644 index 0000000..313b373 Binary files /dev/null and b/deepstream-tracker-3d/figures/.retail_osd.png differ diff --git a/deepstream-tracker-3d/figures/.retail_viz.png b/deepstream-tracker-3d/figures/.retail_viz.png new file mode 100644 index 0000000..2dbebcc Binary files /dev/null and b/deepstream-tracker-3d/figures/.retail_viz.png differ diff --git a/deepstream-tracker-3d/streams/Retail02_short.mp4 b/deepstream-tracker-3d/streams/Retail02_short.mp4 new file mode 100644 index 0000000..a88268b Binary files /dev/null and b/deepstream-tracker-3d/streams/Retail02_short.mp4 differ diff --git a/deepstream-vllm-plugin/README.md b/deepstream-vllm-plugin/README.md new file mode 100644 index 0000000..4888abb --- /dev/null +++ b/deepstream-vllm-plugin/README.md @@ -0,0 +1,284 @@ +# VLLM DeepStream Plugin + +A GStreamer plugin for NVIDIA DeepStream that integrates Vision-Language Models (VLM) using VLLM for real-time video understanding and analysis. + +## Table of Contents + +- [Overview](#overview) +- [Key Features](#key-features) +- [Requirements](#requirements) +- [Quick Start](#quick-start) +- [Configuration](#configuration) +- [Model Support](#model-support) + +## Overview + +The `nvvllmvlm` plugin enables integration of vision-language models (Cosmos-Reason2, etc.) into DeepStream pipelines. It processes video frames in configurable time segments and performs batch VLM inference asynchronously. + +### Key Features + +- **Segment-Based Processing**: Collects frames into time windows for batch inference +- **Flexible Frame Sampling**: FPS-based or interval-based subsampling +- **Async Inference**: Background worker thread for non-blocking processing +- **Multi-Stream Support**: One plugin instance handles multiple streams efficiently +- **Per-Stream Prompts**: Different prompts and settings for each stream +- **Configurable Models**: Supports video-native and image-only VLM models +- **GPU-Optimized**: Zero-copy GPU operations, shared model across streams +- **Flexible Input Formats**: PyTorch tensors, PIL Images, or numpy arrays + +## Requirements + +- NVIDIA GPU with CUDA support +- NVIDIA DeepStream SDK 9.0.0+ +- Docker with NVIDIA Container Runtime +- Python 3.12+ +- 40GB+ GPU memory required +- Currently supported on x86 based GPU platforms + +## Quick Start + +### Installation + +```bash +# Launch DeepStream container +sudo docker run -it --rm --runtime=nvidia --gpus all --network=host \ + -v $(pwd):/home/vllm_ds_plugin \ + nvcr.io/nvidia/deepstream:9.0-triton-multiarch + +# Inside DeepStream container install dependencies +cd /home/vllm_ds_plugin/deepstream-vllm-plugin +./install.sh +``` + +### Steps to get Hugging face Token + +1. Log in at huggingface.co +2. Go to Profile → Access Tokens +3. Create and save the generated token +4. To use cosmos-reason2 model, go to https://huggingface.co/nvidia/Cosmos-Reason2-8B, review and agree Nvidia Open Model License Agreement + +### Single Stream and Multi-Stream Processing +```bash +# Run the following inside DeepStream container: + +# Select GPU (optional) +export CUDA_VISIBLE_DEVICES=0 + +# Export Huggingface token from previous step to download models from HF +export HF_TOKEN= + +# Copy streams to the container or use streams that are already part of the container + +# Single stream (dry-run: results printed to console) +python3 vllm_ds_app_kafka_publish.py --dry-run + +For example, +python3 vllm_ds_app_kafka_publish.py /opt/nvidia/deepstream/deepstream-9.0/samples/streams/sample_1080p_h264.mp4 --dry-run + +# Multi-stream with shared model (dry-run) +python3 vllm_ds_app_kafka_publish.py --dry-run +``` + +### Kafka Integration + +Stream results to Kafka in real-time: +```bash +# Bring up kafka containers by running the following on host outside the DeepStream container: + +# Start Kafka +docker compose -f docker-compose-kafka.yml up -d + +# Run kafka publishing application and consumer script inside DeepStream container: + +# Run with Kafka publishing (single or multi-stream) +python3 vllm_ds_app_kafka_publish.py \ + --kafka-bootstrap localhost:9092 --topic vlm-results + +# On another terminal start the consumer test script +python3 test_consumer.py --topic vlm-results +``` + + +## Configuration + +### Configuration File (config.yaml) + +Place `config.yaml` in the plugin directory or current working directory. + +#### Complete Example + +```yaml +# Model Configuration +model: + path: "nvidia/Cosmos-Reason2-8B" + max_model_len: 20480 # Max context length + gpu_memory_utilization: 0.7 # GPU memory fraction to use. Update depending on platform and availabe gpu memory. + trust_remote_code: true + gpu_id: 0 # GPU device ID (-1 for auto) + + # Video processing mode + video_mode: 1 # 1=video metadata (native video support), 0=multi-image mode + + # Tensor format for image inputs + tensor_format: "pytorch" # pytorch/pil/numpy + +# Segment Processing +segment: + length_sec: 30 # Segment length in seconds + overlap_sec: 0 # Segment overlap in seconds + subsample_interval: 1 # Keep every Nth frame + selection_fps: 30 # Target FPS (0 to disable FPS-based sampling) + +# Inference Configuration +inference: + # User prompt with optional placeholders: {num_frames}, {stream_id}, {timestamps} + # Control timestamp inclusion by using or omitting {timestamps} placeholder + user_prompt: "These are {num_frames} images from stream {stream_id} sampled at timestamps {timestamps}. Describe the scene in detail." + + # System prompt (optional - omit for no system prompt) + system_prompt: | + Provide captions with timestamps using format: + caption of event. + + # Sampling parameters (all optional) + max_tokens: 2048 # Max tokens to generate + temperature: 0.7 # Sampling temperature (0-1) + + # Advanced sampling parameters (optional) + # top_p: 0.9 # Top-p nucleus sampling + # top_k: 100 # Top-k sampling + # repetition_penalty: 1.1 # Repetition penalty + + # Per-stream prompt overrides (multi-stream mode) + stream_prompts: + 0: # Stream 0 + user_prompt: "Stream {stream_id} at {timestamps}: Monitor for security threats." + system_prompt: "You are a security analyst." + 1: # Stream 1 + user_prompt: "Analyze traffic flow." # No {timestamps} = no timestamps + +# Pipeline Configuration +pipeline: + queue_maxsize: 20 # Max inference queue size + max_wait_timeout: 300 # Max wait time for segment completion (seconds) + +# Video Configuration +video: + default_fps_numerator: 30 # Default FPS numerator (30/1 = 30 fps) + default_fps_denominator: 1 # Default FPS denominator (used if stream lacks FPS) +``` + +#### Key Settings + +**Video Mode**: +- `video_mode: 1` - Video metadata (For models that has native video support) +- `video_mode: 0` - Multi-image mode (image-only models) + +**Tensor Format** (image modes only): +- `tensor_format: "pytorch"` - PyTorch tensors (default) +- `tensor_format: "pil"` - PIL Images +- `tensor_format: "numpy"` - numpy arrays + +**System Prompt**: +- Specified: Uses that value +- Omitted: No system prompt (None) +- Empty string `""`: Sends empty system prompt + +**Prompt Placeholders**: +- `{num_frames}` - Number of frames in segment +- `{stream_id}` - Stream identifier +- `{timestamps}` - Timestamp string (e.g., "0.00s 1.00s 2.00s") +- Include `{timestamps}` to show timestamps, omit to exclude them + +**Per-Stream Prompts**: +- Override any inference setting for specific streams +- Streams without overrides use global defaults +- Supports all inference settings per-stream + +#### Plugin Properties + +Properties can override config values at runtime: + +| Property | Type | Default | Description | +|----------|------|---------|-------------| +| `model` | string | from config | HuggingFace model ID | +| `user-prompt` | string | from config | User prompt with placeholders | +| `system-prompt` | string | None | System prompt (optional) | +| `segment-length-sec` | int | 10 | Segment length in seconds | +| `overlap-sec` | int | 0 | Segment overlap | +| `selection-fps` | int | 1 | Target FPS (0=disabled) | +| `subsample-interval` | int | 1 | Keep every Nth frame | +| `max-tokens` | int | 2048 | Max tokens to generate | +| `temperature` | float | 0.7 | Sampling temperature | +| `top-p` | float | 0.9 | Top-p nucleus sampling | +| `top-k` | int | 100 | Top-k sampling | +| `repetition-penalty` | float | 1.1 | Repetition penalty | +| `max-model-len` | int | 20480 | Max model context length | +| `trust-remote-code` | bool | true | Trust remote code | +| `gpu-memory-utilization` | float | 0.7 | GPU memory fraction (0.0-1.0) | +| `gpu-id` | int | 0 | GPU device ID (-1=auto) | +| `video-mode` | int | 1 | 1=video, 0=multi-image | +| `tensor-format` | string | pytorch | pytorch/pil/numpy | +| `queue-maxsize` | int | 20 | Inference queue size | +| `max-wait-timeout` | int | 300 | Shutdown timeout (seconds) | +| `default-fps-numerator` | int | 30 | Default FPS numerator | +| `default-fps-denominator` | int | 1 | Default FPS denominator | + + +## Model Support + +### Supported Models + +**Video-Native Models** (`video_mode: 1`): +- `Cosmos-Reason2-8B` (default) +- Models with native video metadata support + +**Image-Only Models** (`video_mode: 0`): +- Models that only support image inputs + +### Model Configuration Examples + +**Cosmos-Reason2-8B**: +```yaml +model: + path: "nvidia/Cosmos-Reason2-8B" + video_mode: 1 + tensor_format: "pytorch" +``` + +### Custom Prompts with Placeholders + +Control prompt content and timestamps using placeholders: + +**With timestamps**: +```yaml +inference: + user_prompt: "These are {num_frames} images from stream {stream_id} sampled at timestamps {timestamps}. Describe the scene." +``` + +**Without timestamps**: +```yaml +inference: + user_prompt: "Describe what you see in stream {stream_id}." +``` + +**Per-stream custom prompts**: +```yaml +inference: + stream_prompts: + 0: + user_prompt: "Stream {stream_id} at {timestamps}: Security analysis." + 1: + user_prompt: "Analyse vehicles." # Minimal, no placeholders +``` + + +### Signal-Based Results + +Access results via GObject signals: +```python +def on_vlm_result(element, stream_id, start_time, end_time, text, user_data): + print(f"Stream {stream_id} [{start_time:.2f}s-{end_time:.2f}s]: {text}") + +vlm.connect("vlm-result", on_vlm_result, None) +``` diff --git a/deepstream-vllm-plugin/config.yaml b/deepstream-vllm-plugin/config.yaml new file mode 100644 index 0000000..e0efbdc --- /dev/null +++ b/deepstream-vllm-plugin/config.yaml @@ -0,0 +1,107 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### +# +# Configuration for gstnvvllmvlm.py +# VLM Plugin Configuration + +# Model Configuration +model: + # HuggingFace model path/ID + path: "nvidia/Cosmos-Reason2-8B" + + # Maximum model context length + max_model_len: 20480 + + # Trust remote code when loading model + trust_remote_code: true + + # GPU memory utilization (0.0 to 1.0) + gpu_memory_utilization: 0.7 + + # GPU device ID to use (-1 for auto from CUDA_VISIBLE_DEVICES, 0-15 for specific GPU) + gpu_id: 0 + + # Video mode: 1 = use video metadata (multi-frame), 0 = image-only mode (to pass frames as images to VLLM) + # Set to 0 for models that don't support video + # Set to 1 for models that support video + video_mode: 1 + + # Tensor format for image inputs: "pytorch", "pil", or "numpy" + # Note: Video mode (video_mode=1, multi-frame) always uses PyTorch tensors + # Image modes can use any format based on model requirements + tensor_format: "pytorch" + +# Segment Processing Configuration +segment: + # Length of each segment in seconds + length_sec: 10 + + # Overlap between consecutive segments in seconds + overlap_sec: 0 + + # Keep every Nth frame (subsample interval) + subsample_interval: 1 + + # Target frames per second per segment (0 = disabled, use subsample_interval instead) + selection_fps: 1 + +# Inference Configuration +inference: + # User prompt with optional placeholders: {num_frames}, {stream_id}, {timestamps} + # Control timestamp inclusion by using or omitting {timestamps} placeholder + user_prompt: "Describe what you see in detail" + + # System prompt (optional - omit for no system prompt) + # system_prompt: + + # Sampling parameters (all optional) + # Max tokens + max_tokens: 2048 + + # Temperature + temperature: 0.7 + + # Top-p (nucleus) sampling + top_p: 0.9 + + # Top-k sampling + top_k: 100 + + # Repetition penalty + repetition_penalty: 1.1 + + # Per-stream prompt overrides (optional) + # stream_prompts: + # 0: # Stream 0 + # user_prompt: "Frames are sampled at {timestamps}: Monitor for warehouse safety hazards." + # system_prompt: "You are a warehouse analyst." + # 1: # Stream 1 + # user_prompt: "Analyse traffic flow" # No {timestamps} = no timestamps + +# Pipeline Configuration +pipeline: + # Maximum size of inference queue + queue_maxsize: 20 + + # Maximum wait time for segment completion during shutdown (seconds) + max_wait_timeout: 300 + +# Video Configuration +video: + # Default FPS if not detected from stream + default_fps_numerator: 30 + default_fps_denominator: 1 diff --git a/deepstream-vllm-plugin/config_loader.py b/deepstream-vllm-plugin/config_loader.py new file mode 100644 index 0000000..73c4fb6 --- /dev/null +++ b/deepstream-vllm-plugin/config_loader.py @@ -0,0 +1,200 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### + +"""Configuration loader for VLM plugin""" + +import os +from pathlib import Path +from typing import Any, Dict, Optional + +import yaml + + +class Config: + """Configuration manager for VLM plugin""" + + def __init__(self, config_path: Optional[str] = None): + """ + Load configuration from YAML file + + Args: + config_path: Path to config file. If None, searches for + config.yaml in: + 1. Current directory + 2. Script directory + 3. Uses default values + """ + self._config = self._load_config(config_path) + + def _load_config(self, config_path: Optional[str]) -> Dict[str, Any]: + """Load config from file or use defaults""" + + # Try to find config file + if config_path is None: + # Search in common locations + search_paths = [ + Path.cwd() / "config.yaml", + Path(__file__).parent / "config.yaml", + ] + + for path in search_paths: + if path.exists(): + config_path = str(path) + break + + # Load from file if found + if config_path and os.path.exists(config_path): + with open(config_path, "r") as f: + return yaml.safe_load(f) + + # Return empty dict if no config found (will use property defaults) + return {} + + # Model properties + @property + def model_path(self) -> str: + return self._config.get("model", {}).get( # noqa: BLK100 + "path", "nvidia/Cosmos-Reason2-8B" + ) + + @property + def max_model_len(self) -> int: + return self._config.get("model", {}).get("max_model_len", 20480) + + @property + def trust_remote_code(self) -> bool: + return self._config.get("model", {}).get("trust_remote_code", True) + + @property + def gpu_memory_utilization(self) -> float: + return self._config.get("model", {}).get("gpu_memory_utilization", 0.4) + + @property + def gpu_id(self) -> int: + return self._config.get("model", {}).get("gpu_id", 0) + + @property + def video_mode(self) -> int: + return self._config.get("model", {}).get("video_mode", 1) + + @property + def tensor_format(self) -> str: + return self._config.get("model", {}).get("tensor_format", "pytorch") + + # Segment properties + @property + def segment_length_sec(self) -> int: + return self._config.get("segment", {}).get("length_sec", 10) + + @property + def overlap_sec(self) -> int: + return self._config.get("segment", {}).get("overlap_sec", 0) + + @property + def subsample_interval(self) -> int: + return self._config.get("segment", {}).get("subsample_interval", 1) + + @property + def selection_fps(self) -> int: + return self._config.get("segment", {}).get("selection_fps", 1) + + # Inference properties + @property + def user_prompt(self) -> str: + return self._config.get("inference", {}).get( + "user_prompt", "Describe the scene in detail." + ) + + @property + def system_prompt(self) -> str: + # Return None if not specified in config (no default system prompt) + return self._config.get("inference", {}).get("system_prompt", None) + + @property + def max_tokens(self) -> int: + """Max tokens default: 2048""" + return self._config.get("inference", {}).get("max_tokens", 2048) + + @property + def temperature(self) -> float: + """Temperature default: 0.7""" + return self._config.get("inference", {}).get("temperature", 0.7) + + @property + def top_p(self) -> Optional[float]: + """Top-p (nucleus) sampling parameter""" + return self._config.get("inference", {}).get("top_p", None) + + @property + def top_k(self) -> Optional[int]: + """Top-k sampling parameter""" + value = self._config.get("inference", {}).get("top_k", None) + return int(value) if value is not None else None + + @property + def repetition_penalty(self) -> Optional[float]: + """Repetition penalty parameter""" + return self._config.get("inference", {}).get( # noqa: BLK100 + "repetition_penalty", None + ) + + @property + def stream_prompts(self) -> dict: + """ + Get per-stream prompt overrides + Returns dict: {stream_id: {setting: value, ...}} + """ + return self._config.get("inference", {}).get("stream_prompts", {}) + + # Pipeline properties + @property + def queue_maxsize(self) -> int: + return self._config.get("pipeline", {}).get("queue_maxsize", 20) + + @property + def max_wait_timeout(self) -> int: + return self._config.get("pipeline", {}).get("max_wait_timeout", 300) + + # Video properties + @property + def default_fps(self) -> tuple: + numerator = self._config.get("video", {}).get( # noqa: BLK100 + "default_fps_numerator", 30 + ) + denominator = self._config.get("video", {}).get( # noqa: BLK100 + "default_fps_denominator", 1 + ) + return (numerator, denominator) + + +# Global config instance +_config_instance: Optional[Config] = None + + +def get_config(config_path: Optional[str] = None) -> Config: + """Get or create global config instance""" + global _config_instance + if _config_instance is None: + _config_instance = Config(config_path) + return _config_instance + + +def reload_config(config_path: Optional[str] = None) -> Config: + """Reload configuration from file""" + global _config_instance + _config_instance = Config(config_path) + return _config_instance diff --git a/deepstream-vllm-plugin/docker-compose-kafka.yml b/deepstream-vllm-plugin/docker-compose-kafka.yml new file mode 100644 index 0000000..2db2f1c --- /dev/null +++ b/deepstream-vllm-plugin/docker-compose-kafka.yml @@ -0,0 +1,43 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### +version: '3.8' + +# RECOMMENDED: Use host networking for compatibility with DeepStream containers +# This allows the DeepStream container to access Kafka on localhost:9092 + +services: + zookeeper: + image: confluentinc/cp-zookeeper:7.5.0 + container_name: zookeeper + network_mode: host + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + kafka: + image: confluentinc/cp-kafka:7.5.0 + container_name: kafka + network_mode: host + depends_on: + - zookeeper + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: localhost:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" diff --git a/deepstream-vllm-plugin/gstnvvllmvlm.py b/deepstream-vllm-plugin/gstnvvllmvlm.py new file mode 100644 index 0000000..8b336c3 --- /dev/null +++ b/deepstream-vllm-plugin/gstnvvllmvlm.py @@ -0,0 +1,1485 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### + +import gi + +gi.require_version("Gst", "1.0") +gi.require_version("GstBase", "1.0") +import multiprocessing as mp # noqa: E402 +import threading # noqa: E402 +from queue import Empty, Queue # noqa: E402 +from typing import Any, Dict, List, Optional # noqa: E402 + +import torch # noqa: E402 +from gi.repository import GObject, Gst, GstBase # noqa: E402 +from PIL import Image # noqa: E402 +from pyservicemaker import Buffer # noqa: E402 +from transformers import AutoTokenizer # noqa: E402 +from vllm import LLM, SamplingParams # noqa: E402 + +# Set multiprocessing start method for CUDA compatibility +try: + mp.set_start_method("spawn", force=True) +except RuntimeError: + pass # Already set + +# Load configuration +from config_loader import get_config # noqa: E402 + +Gst.init(None) + +GST_PLUGIN_NAME = "nvvllmvlm" + +# Load config instance +config = get_config() + +# Configuration values (can be overridden via config.yaml) +MODEL_PATH = config.model_path +DEFAULT_SEGMENT_LEN_SEC = config.segment_length_sec +DEFAULT_OVERLAP_SEC = config.overlap_sec +DEFAULT_SUBSAMPLE_INTERVAL = config.subsample_interval +DEFAULT_SELECTION_FPS = config.selection_fps + + +class BufferData: + """Frame data for VLM processing""" + + def __init__( + self, + frame_number: int, + pts: int, + dts: int, + duration: int, + tensor_gpu: torch.Tensor, + ) -> None: + self.frame_number = frame_number + self.pts = pts + self.tensor_gpu = tensor_gpu + + +class Segment: + """Temporal segment containing multiple frames""" + + def __init__( + self, stream_id: int, start_pts_ns: int, end_pts_ns: int, batch_id: int + ) -> None: + self.stream_id = stream_id # Which stream this segment belongs to + self.start_pts_ns = start_pts_ns + self.end_pts_ns = end_pts_ns + self.batch_id = batch_id + self.frames: List[BufferData] = [] + self.last_saved_pts_ns: Optional[int] = None + + +class StreamContext: + """Per-stream context for multi-stream processing""" + + def __init__(self, stream_id: int): + self.stream_id = stream_id + + # Segment management + self.open_segments: List[Segment] = [] + self.next_segment_start_pts: Optional[int] = None + self.base_pts: Optional[int] = None + self.frame_counter: int = 0 + + # Statistics + self.segments_submitted: int = 0 + self.segments_completed: int = 0 + self.segments_dropped: int = 0 + self.total_frames_in_segments: int = 0 + + # Latest result + self.latest_text: Optional[str] = None + self.latest_lock: threading.Lock = threading.Lock() + + def update_result(self, text: str, start_sec: float, end_sec: float): + """Update the latest result for this stream""" + with self.latest_lock: + self.latest_text = ( + f"[Stream {self.stream_id}] " + f"[{start_sec:.2f}s-{end_sec:.2f}s] {text}" + ) + self.segments_completed += 1 + + +class SegmentRequest: + """Request for VLM inference on a segment""" + + def __init__(self, stream_id: int, segment: Segment, prompt_config: Dict): + self.stream_id = stream_id + self.segment = segment + self.prompt_config = prompt_config + + +class NvVllmVLM(GstBase.BaseTransform): + __gstmetadata__ = ( + "NvVllmVLM", + "Generic/Analyzer", + "vLLM inference with multi-stream support", + "VSS", + ) + + src_format = Gst.Caps.from_string( + "video/x-raw(memory:NVMM), format=RGB, " + "width=(int)[ 1, 2147483647 ], height=(int)[ 1, 2147483647 ], " + "framerate=(fraction)[ 0/1, 2147483647/1 ]" + ) + sink_format = Gst.Caps.from_string( + "video/x-raw(memory:NVMM), format=RGB, " + "width=(int)[ 1, 2147483647 ], height=(int)[ 1, 2147483647 ], " + "framerate=(fraction)[ 0/1, 2147483647/1 ]" + ) + + src_pad_template = Gst.PadTemplate.new( + "src", Gst.PadDirection.SRC, Gst.PadPresence.ALWAYS, src_format + ) + sink_pad_template = Gst.PadTemplate.new( + "sink", Gst.PadDirection.SINK, Gst.PadPresence.ALWAYS, sink_format + ) + __gsttemplates__ = (src_pad_template, sink_pad_template) + + __gsignals__ = { + "vlm-result": ( + GObject.SignalFlags.RUN_LAST, + None, + ( + int, + float, + float, + str, + ), + ) + # Signal emitted when VLM inference completes for a segment + # Args: stream_id (int), start_time (float), end_time (float), + # result_text (str) + } + + __gproperties__ = { + "segment-length-sec": ( + int, + "Segment Length (sec)", + "Length of each segment in seconds", + 1, + 3600, + DEFAULT_SEGMENT_LEN_SEC, + GObject.ParamFlags.READWRITE, + ), + "overlap-sec": ( + int, + "Overlap (sec)", + "Overlap between consecutive segments", + -600, + 600, + DEFAULT_OVERLAP_SEC, + GObject.ParamFlags.READWRITE, + ), + "subsample-interval": ( + int, + "Subsample Interval", + "Keep every Nth frame", + 1, + 100, + DEFAULT_SUBSAMPLE_INTERVAL, + GObject.ParamFlags.READWRITE, + ), + "selection-fps": ( + int, + "Selection FPS", + "Target frames per second per segment (0 = disabled)", + 0, + 240, + DEFAULT_SELECTION_FPS, + GObject.ParamFlags.READWRITE, + ), + "model": ( + str, + "Model", + "HuggingFace model id", + MODEL_PATH, + GObject.ParamFlags.READWRITE, + ), + "user-prompt": ( + str, + "User Prompt", + "User prompt with optional placeholders: " + "{num_frames}, {stream_id}, {timestamps}", + "Describe what you see in detail", + GObject.ParamFlags.READWRITE, + ), + "max-tokens": ( + int, + "Max Tokens", + "Maximum tokens to generate", + 1, + 8192, + 2048, + GObject.ParamFlags.READWRITE, + ), + "temperature": ( + float, + "Temperature", + "Sampling temperature", + 0.0, + 2.0, + 0.7, + GObject.ParamFlags.READWRITE, + ), + "gpu-id": ( + int, + "GPU ID", + ( + "GPU device ID to use " + "(default: 0, -1 = auto from CUDA_VISIBLE_DEVICES)" + ), + -1, + 15, + 0, + GObject.ParamFlags.READWRITE, + ), + "video-mode": ( + int, + "Video Mode", + "Video mode (1=video metadata, 0=image-only)", + 0, + 1, + 1, + GObject.ParamFlags.READWRITE, + ), + "tensor-format": ( + str, + "Tensor Format", + "Image tensor format: pytorch, pil, or numpy", + "pytorch", + GObject.ParamFlags.READWRITE, + ), + "top-p": ( + float, + "Top P", + "Top-p nucleus sampling", + 0.0, + 1.0, + 0.9, + GObject.ParamFlags.READWRITE, + ), + "top-k": ( + int, + "Top K", + "Top-k sampling", + -1, + 1000, + 100, + GObject.ParamFlags.READWRITE, + ), + "repetition-penalty": ( + float, + "Repetition Penalty", + "Repetition penalty", + 1.0, + 2.0, + 1.1, + GObject.ParamFlags.READWRITE, + ), + "max-model-len": ( + int, + "Max Model Length", + "Maximum model context length", + 512, + 65536, + 20480, + GObject.ParamFlags.READWRITE, + ), + "trust-remote-code": ( + bool, + "Trust Remote Code", + "Trust remote code when loading model", + True, + GObject.ParamFlags.READWRITE, + ), + "gpu-memory-utilization": ( + float, + "GPU Memory Utilization", + "GPU memory fraction (0.0 to 1.0)", + 0.1, + 1.0, + 0.7, + GObject.ParamFlags.READWRITE, + ), + "system-prompt": ( + str, + "System Prompt", + "System prompt for inference (optional)", + None, + GObject.ParamFlags.READWRITE, + ), + "queue-maxsize": ( + int, + "Queue Maxsize", + "Maximum size of inference queue", + 1, + 1000, + 20, + GObject.ParamFlags.READWRITE, + ), + "max-wait-timeout": ( + int, + "Max Wait Timeout", + "Maximum wait time for segment completion (seconds)", + 1, + 3600, + 300, + GObject.ParamFlags.READWRITE, + ), + "default-fps-numerator": ( + int, + "Default FPS Numerator", + "Default FPS numerator if not detected from stream", + 1, + 240, + 30, + GObject.ParamFlags.READWRITE, + ), + "default-fps-denominator": ( + int, + "Default FPS Denominator", + "Default FPS denominator if not detected from stream", + 1, + 1000, + 1, + GObject.ParamFlags.READWRITE, + ), + } + + def __init__(self) -> None: + GstBase.BaseTransform.__init__(self) + + # Segment configuration + self.segment_length_sec: int = DEFAULT_SEGMENT_LEN_SEC + self.overlap_sec: int = DEFAULT_OVERLAP_SEC + self.subsample_interval: int = DEFAULT_SUBSAMPLE_INTERVAL + self.selection_fps: int = DEFAULT_SELECTION_FPS + + # Video format info + self.width: Optional[int] = None + self.height: Optional[int] = None + self.format: Optional[str] = None + self.fps: Optional[tuple] = None # (numerator, denominator) + + # Computed values + self._step_ns: int = self._compute_step_ns() + self._seg_len_ns: int = self.segment_length_sec * 1_000_000_000 + self._sample_interval_ns: Optional[int] = None + + # Model configuration (load from config) + self.model: str = MODEL_PATH + self.user_prompt: str = config.user_prompt + self.max_tokens: int = config.max_tokens + self.temperature: float = config.temperature + self.gpu_id: int = config.gpu_id + self.video_mode: int = config.video_mode + self.tensor_format: str = config.tensor_format + # Can be None if not specified + self._system_prompt = config.system_prompt + + # Additional sampling parameters (optional) + self.top_p: Optional[float] = config.top_p + self.top_k: Optional[int] = config.top_k + self.repetition_penalty: Optional[float] = config.repetition_penalty + + # Model initialization parameters + self.max_model_len: int = config.max_model_len + self.trust_remote_code: bool = config.trust_remote_code + self.gpu_memory_utilization: float = config.gpu_memory_utilization + + # Pipeline parameters + self.queue_maxsize: int = config.queue_maxsize + self.max_wait_timeout: int = config.max_wait_timeout + + # Video parameters + self.default_fps_numerator: int = config.default_fps[0] + self.default_fps_denominator: int = config.default_fps[1] + + # Per-stream prompt overrides + self._stream_prompts: Dict[int, Dict[str, Any]] = config.stream_prompts + + # Single VLM model instance (shared across all streams naturally) + self.llm: Optional[LLM] = None + self.tokenizer: Optional[AutoTokenizer] = None + + # Per-stream contexts (keyed by pad_index/source_id) + self.stream_contexts: Dict[int, StreamContext] = {} + self.stream_contexts_lock: threading.Lock = threading.Lock() + + # Shared inference queue for all streams + self._infer_queue: Queue = Queue(maxsize=self.queue_maxsize) + self._infer_thread: Optional[threading.Thread] = None + self._stop_event: threading.Event = threading.Event() + + # Load model once + try: + Gst.info(f"{GST_PLUGIN_NAME}: Loading VLM model '{self.model}'") + + # Initialize CUDA + if torch.cuda.is_available(): + torch.cuda.init() + + # Determine which GPU to use + if self.gpu_id >= 0: + # Explicit GPU ID specified + if self.gpu_id >= torch.cuda.device_count(): + Gst.warning( + f"{GST_PLUGIN_NAME}: Requested GPU " + f"{self.gpu_id} not available. Using GPU 0 " + f"(available: {torch.cuda.device_count()} GPUs)" + ) + self.gpu_id = 0 + torch.cuda.set_device(self.gpu_id) + device_name = torch.cuda.get_device_name(self.gpu_id) + Gst.info( + f"{GST_PLUGIN_NAME}: CUDA initialized on GPU " + f"{self.gpu_id} ({device_name})" + ) + else: + # Auto-select from CUDA_VISIBLE_DEVICES + # (use device 0 of visible devices) + torch.cuda.set_device(0) + device_name = torch.cuda.get_device_name(0) + Gst.info( + f"{GST_PLUGIN_NAME}: CUDA initialized on GPU 0 " + f"(auto from CUDA_VISIBLE_DEVICES, {device_name})" + ) + else: + Gst.error(f"{GST_PLUGIN_NAME}: CUDA not available!") + raise RuntimeError("CUDA not available") + + self.llm = LLM( + model=self.model, + max_model_len=self.max_model_len, + trust_remote_code=self.trust_remote_code, + gpu_memory_utilization=self.gpu_memory_utilization, + ) + try: + self.tokenizer = AutoTokenizer.from_pretrained( + self.model, trust_remote_code=self.trust_remote_code + ) + except Exception: + self.tokenizer = None + + Gst.info(f"{GST_PLUGIN_NAME}: VLM model loaded successfully") + except Exception as e: + Gst.error(f"{GST_PLUGIN_NAME}: Failed to initialize vLLM - {e}") + import traceback + + traceback.print_exc() + + def _compute_step_ns(self) -> int: + step = max(1, (self.segment_length_sec - self.overlap_sec)) + return step * 1_000_000_000 + + def _update_sample_interval(self) -> None: + if self.selection_fps and self.selection_fps > 0: + self._sample_interval_ns = int(1_000_000_000 / self.selection_fps) + Gst.info( + f"{GST_PLUGIN_NAME}: selection_fps={self.selection_fps}, " + f"sample_interval_ns={self._sample_interval_ns}" + ) + else: + self._sample_interval_ns = None + Gst.info( + f"{GST_PLUGIN_NAME}: selection_fps disabled, using " + f"subsample-interval={self.subsample_interval}" + ) + + def do_get_property(self, prop: GObject.ParamSpec) -> Any: + if prop.name == "model": + return self.model + if prop.name == "user-prompt": + return self.user_prompt + if prop.name == "max-tokens": + return self.max_tokens + if prop.name == "temperature": + return self.temperature + if prop.name == "gpu-id": + return self.gpu_id + if prop.name == "video-mode": + return self.video_mode + if prop.name == "tensor-format": + return self.tensor_format + if prop.name == "segment-length-sec": + return self.segment_length_sec + if prop.name == "overlap-sec": + return self.overlap_sec + if prop.name == "subsample-interval": + return self.subsample_interval + if prop.name == "selection-fps": + return self.selection_fps + if prop.name == "top-p": + return self.top_p + if prop.name == "top-k": + return self.top_k + if prop.name == "repetition-penalty": + return self.repetition_penalty + if prop.name == "max-model-len": + return self.max_model_len + if prop.name == "trust-remote-code": + return self.trust_remote_code + if prop.name == "gpu-memory-utilization": + return self.gpu_memory_utilization + if prop.name == "system-prompt": + return self._system_prompt + if prop.name == "queue-maxsize": + return self.queue_maxsize + if prop.name == "max-wait-timeout": + return self.max_wait_timeout + if prop.name == "default-fps-numerator": + return self.default_fps_numerator + if prop.name == "default-fps-denominator": + return self.default_fps_denominator + msg = f"{GST_PLUGIN_NAME}: Unknown property '{prop.name}'" + raise AttributeError(msg) + + def do_set_property(self, prop: GObject.ParamSpec, value: Any) -> None: + if prop.name == "model": + self.model = value + elif prop.name == "user-prompt": + self.user_prompt = value + elif prop.name == "max-tokens": + self.max_tokens = value + elif prop.name == "temperature": + self.temperature = value + elif prop.name == "gpu-id": + self.gpu_id = int(value) + elif prop.name == "video-mode": + self.video_mode = int(value) + elif prop.name == "tensor-format": + self.tensor_format = str(value).lower() + if self.tensor_format not in ["pytorch", "pil", "numpy"]: + Gst.warning( + f"{GST_PLUGIN_NAME}: Invalid tensor-format '{value}', " + f"using 'pytorch'" + ) + self.tensor_format = "pytorch" + elif prop.name == "segment-length-sec": + self.segment_length_sec = int(value) + self._seg_len_ns = self.segment_length_sec * 1_000_000_000 + self._step_ns = self._compute_step_ns() + elif prop.name == "overlap-sec": + self.overlap_sec = int(value) + self._step_ns = self._compute_step_ns() + elif prop.name == "subsample-interval": + self.subsample_interval = max(1, int(value)) + elif prop.name == "selection-fps": + self.selection_fps = max(0, int(value)) + self._update_sample_interval() + elif prop.name == "top-p": + self.top_p = float(value) if value is not None else None + elif prop.name == "top-k": + self.top_k = int(value) if value is not None else None + elif prop.name == "repetition-penalty": + self.repetition_penalty = ( # noqa: BLK100 + float(value) if value is not None else None + ) + elif prop.name == "max-model-len": + self.max_model_len = int(value) + elif prop.name == "trust-remote-code": + self.trust_remote_code = bool(value) + elif prop.name == "gpu-memory-utilization": + self.gpu_memory_utilization = float(value) + elif prop.name == "system-prompt": + self._system_prompt = str(value) if value is not None else None + elif prop.name == "queue-maxsize": + self.queue_maxsize = int(value) + elif prop.name == "max-wait-timeout": + self.max_wait_timeout = int(value) + elif prop.name == "default-fps-numerator": + self.default_fps_numerator = int(value) + elif prop.name == "default-fps-denominator": + self.default_fps_denominator = int(value) + else: + msg = f"{GST_PLUGIN_NAME}: Unknown property '{prop.name}'" + raise AttributeError(msg) + + def do_start(self) -> bool: + if self.llm is None: + Gst.error(f"{GST_PLUGIN_NAME}: vLLM not initialized") + return False + + # Start inference worker thread + self._stop_event.clear() + self._infer_thread = threading.Thread( + target=self._inference_worker, name="vlm-worker", daemon=True + ) + self._infer_thread.start() + + self._update_sample_interval() + msg = f"{GST_PLUGIN_NAME}: Plugin started - ready for multi-stream" + Gst.info(msg) + return True + + def do_set_caps(self, incaps: Gst.Caps, outcaps: Gst.Caps) -> bool: + struct = incaps.get_structure(0) + self.width = struct.get_int("width").value + self.height = struct.get_int("height").value + self.format = struct.get_string("format") + + # get_fraction returns (success, numerator, denominator) + fps_result = struct.get_fraction("framerate") + if fps_result[0]: # success + self.fps = ( + fps_result[1], + fps_result[2], + ) # Store as tuple (numerator, denominator) + Gst.info( + f"{GST_PLUGIN_NAME}: caps set - {self.width}x{self.height} " + f"format={self.format} fps={self.fps[0]}/{self.fps[1]}" + ) + else: + self.fps = ( + self.default_fps_numerator, + self.default_fps_denominator, + ) + Gst.info( + f"{GST_PLUGIN_NAME}: caps set - {self.width}x{self.height} " + f"format={self.format} fps={self.fps[0]}/{self.fps[1]} " + f"(default)" + ) + + return True + + def _get_or_create_stream_context(self, stream_id: int) -> StreamContext: + """Get or create context for a stream""" + with self.stream_contexts_lock: + if stream_id not in self.stream_contexts: + ctx = StreamContext(stream_id) + self.stream_contexts[stream_id] = ctx + Gst.info( + f"{GST_PLUGIN_NAME}: Created context for stream " + f"{stream_id} (total streams: {len(self.stream_contexts)})" + ) + return self.stream_contexts[stream_id] + + def _ensure_segments_until( + self, ctx: StreamContext, pts_ns: int, batch_id: int + ) -> None: + """Create segments for a stream until covering pts_ns""" + if ctx.next_segment_start_pts is None: + ctx.next_segment_start_pts = pts_ns + + while ( + ctx.next_segment_start_pts is not None + and ctx.next_segment_start_pts <= pts_ns + ): + start = ctx.next_segment_start_pts + end = start + self._seg_len_ns + seg = Segment(ctx.stream_id, start, end, batch_id) + ctx.open_segments.append(seg) + Gst.debug( + f"{GST_PLUGIN_NAME}[Stream {ctx.stream_id}]: " + f"opened segment [{start/1e9:.2f}s - {end/1e9:.2f}s]" + ) + ctx.next_segment_start_pts = start + self._step_ns + + def _finalize_segments_up_to( + self, ctx: StreamContext, pts_ns: int, batch_id: int + ) -> None: + """Finalize completed segments for a stream""" + to_finalize = [] + for s in ctx.open_segments: + batch_match = s.batch_id == batch_id or batch_id is None + if s.end_pts_ns <= pts_ns and batch_match: + to_finalize.append(s) + + # Determine if we're in multi-stream mode for cleaner logging + num_streams = len(self.stream_contexts) + stream_label = f"[Stream {ctx.stream_id}]" if num_streams > 1 else "" + + for seg in to_finalize: + if seg.frames: + start_sec = seg.start_pts_ns / 1_000_000_000 + end_sec = seg.end_pts_ns / 1_000_000_000 + print( + f"{GST_PLUGIN_NAME}{stream_label}: Finalizing segment " + f"[{start_sec:.2f}s - {end_sec:.2f}s] " + f"with {len(seg.frames)} frames" + ) + + # Build stream-specific prompt config with fallback to global + prompt_config = { + "user_prompt": self._get_stream_config( + ctx.stream_id, "user_prompt", self.user_prompt + ), + "system_prompt": self._get_stream_config( + ctx.stream_id, "system_prompt", self._system_prompt + ), + "max_tokens": self._get_stream_config( + ctx.stream_id, "max_tokens", self.max_tokens + ), + "temperature": self._get_stream_config( + ctx.stream_id, "temperature", self.temperature + ), + } + + # Add optional sampling parameters if specified + top_p = self._get_stream_config( # noqa: BLK100 + ctx.stream_id, "top_p", self.top_p + ) + if top_p is not None: + prompt_config["top_p"] = top_p + + top_k = self._get_stream_config( # noqa: BLK100 + ctx.stream_id, "top_k", self.top_k + ) + if top_k is not None: + prompt_config["top_k"] = top_k + + repetition_penalty = self._get_stream_config( + ctx.stream_id, + "repetition_penalty", + self.repetition_penalty, + ) + if repetition_penalty is not None: + prompt_config["repetition_penalty"] = repetition_penalty + + request = SegmentRequest(ctx.stream_id, seg, prompt_config) + try: + self._infer_queue.put_nowait(request) + ctx.segments_submitted += 1 + ctx.total_frames_in_segments += len(seg.frames) + print( + f"{GST_PLUGIN_NAME}{stream_label}: Submitted " + f"(total: {ctx.segments_submitted})" + ) + except Exception as e: + ctx.segments_dropped += 1 + print(f"{GST_PLUGIN_NAME}{stream_label}: Dropped - {e}") + + try: + ctx.open_segments.remove(seg) + except ValueError: + pass + + def do_transform_ip(self, gst_buffer: Gst.Buffer) -> Gst.FlowReturn: + """Process batched frames from multiple streams""" + + buffer = Buffer(gst_buffer) + batch_meta = buffer.batch_meta + + if batch_meta.n_frames == 0: + return Gst.FlowReturn.OK + + # Process each frame in the batch + for frame_meta in batch_meta.frame_items: + try: + # Identify which stream this frame belongs to + stream_id = frame_meta.pad_index # or frame_meta.source_id + + # Get or create context for this stream + ctx = self._get_or_create_stream_context(stream_id) + + # Process frame for this stream's context + ctx.frame_counter += 1 + subsample_mod = ctx.frame_counter % self.subsample_interval + keep_by_subsample = subsample_mod == 0 + + current_pts = frame_meta.buffer_pts + + # Ensure segments and finalize completed ones + batch_id = frame_meta.batch_id + self._ensure_segments_until(ctx, current_pts, batch_id) + prev_pts = current_pts - 1 + self._finalize_segments_up_to(ctx, prev_pts, batch_id) + + # Extract frame data + tensor = buffer.extract(batch_id) + torch_frame_data = torch.utils.dlpack.from_dlpack(tensor) + + if torch_frame_data is None or torch_frame_data.numel() == 0: + continue + + bd_created = False + bd = None + + # Add frame to appropriate segments for this stream + if ( + self._sample_interval_ns is not None + and self._sample_interval_ns > 0 + ): + # FPS-based sampling + for seg in ctx.open_segments: + if seg.batch_id != frame_meta.batch_id: + continue + if seg.start_pts_ns <= current_pts <= seg.end_pts_ns: + should_keep = ( + seg.last_saved_pts_ns is None + or (current_pts - seg.last_saved_pts_ns) + >= self._sample_interval_ns + ) + if should_keep: + if not bd_created: + bd = BufferData( + -1, + current_pts, + -1, + -1, + torch_frame_data.clone(), + ) + bd_created = True + seg.frames.append(bd) + seg.last_saved_pts_ns = current_pts + else: + # Interval-based sampling + if keep_by_subsample: + bd = BufferData( + -1, current_pts, -1, -1, torch_frame_data.clone() + ) + for seg in ctx.open_segments: + start = seg.start_pts_ns + end = seg.end_pts_ns + if start <= current_pts <= end: + seg.frames.append(bd) + seg.last_saved_pts_ns = current_pts + + except Exception as e: + msg = f"{GST_PLUGIN_NAME}: Frame processing failed - {e}" + Gst.warning(msg) + + return Gst.FlowReturn.OK + + def _get_stream_config( # noqa: BLK100 + self, stream_id: int, setting: str, default: Any + ) -> Any: + """ + Get config value for a specific stream with fallback to global + + Priority: + 1. Stream-specific setting (if exists in stream_prompts) + 2. Global setting (self.setting) + 3. Default value + + Args: + stream_id: Stream identifier + setting: Setting name (e.g., 'user_prompt', 'system_prompt') + default: Default value if not found + + Returns: + Config value for this stream + """ + # Check if stream has specific override + if ( + stream_id in self._stream_prompts + and setting in self._stream_prompts[stream_id] + ): + value = self._stream_prompts[stream_id][setting] + Gst.debug( + f"{GST_PLUGIN_NAME}[Stream {stream_id}]: " + f"Using stream-specific {setting}: {value}" + ) + return value + + # Fall back to global setting or default + return default + + def _format_user_prompt( + self, + user_prompt: str, + stream_id: int, + num_frames: int, + timestamps: str, + ) -> str: + """ + Format user prompt by replacing placeholders. + + Available placeholders: + - {num_frames}: Number of frames in segment + - {stream_id}: Stream identifier + - {timestamps}: Timestamp string (e.g., "0.00s 1.00s 2.00s") + + Args: + user_prompt: User's prompt string with placeholders + stream_id: Stream ID + num_frames: Number of frames + timestamps: Timestamp string + + Returns: + Formatted prompt string + """ + try: + return user_prompt.format( + num_frames=num_frames, + stream_id=stream_id, + timestamps=timestamps, + ) + except KeyError as e: + msg = f"{GST_PLUGIN_NAME}: Invalid placeholder in user_prompt: {e}" + Gst.warning(msg) + return user_prompt + + def _convert_tensor_to_format( + self, tensor: torch.Tensor, target_format: str + ): # noqa: BLK100 + """ + Convert PyTorch tensor to specified format (pytorch, pil, or numpy) + + Args: + tensor: PyTorch tensor with shape [C, H, W], RGB format + target_format: "pytorch", "pil", or "numpy" + + Returns: + Converted tensor in requested format + """ + if target_format == "pytorch": + return tensor.cpu() + + elif target_format == "pil": + # Convert [C, H, W] to [H, W, C] for PIL + tensor = tensor.cpu() + + # Handle different dtypes + if tensor.dtype in (torch.float32, torch.float16): + # Assume normalized [0, 1], convert to [0, 255] uint8 + tensor = (tensor * 255).clamp(0, 255).byte() + elif tensor.dtype == torch.uint8: + # Already uint8 + pass + else: + # Convert to uint8 + tensor = tensor.byte() + + # Convert to numpy and create PIL Image + # [C, H, W] -> [H, W, C] + np_array = tensor.permute(1, 2, 0).numpy() + return Image.fromarray(np_array, mode="RGB") + + elif target_format == "numpy": + # Convert to numpy, keep shape [C, H, W] + return tensor.cpu().numpy() + + else: + Gst.warning( + f"{GST_PLUGIN_NAME}: Unknown tensor format " + f"'{target_format}', using pytorch" + ) + return tensor.cpu() + + def _inference_worker(self) -> None: + """Worker thread processes segments from all streams""" + print(f"{GST_PLUGIN_NAME}: Inference worker started (multi-stream)") + + while not self._stop_event.is_set(): + try: + request = self._infer_queue.get(timeout=0.1) + except Empty: + continue + + stream_id = request.stream_id + segment = request.segment + start_sec = segment.start_pts_ns / 1_000_000_000 + end_sec = segment.end_pts_ns / 1_000_000_000 + + # Determine stream label based on number of active streams + num_streams = len(self.stream_contexts) + stream_label = f"[Stream {stream_id}]" if num_streams > 1 else "" + + msg = ( + f"{GST_PLUGIN_NAME}{stream_label}: Processing segment " + f"[{start_sec:.2f}s - {end_sec:.2f}s] " + f"with {len(segment.frames)} frames" + ) + print(msg) + + try: + result_text = self._run_vlm_batch( + segment, request.prompt_config + ) # noqa: BLK100 + if result_text: + ctx = self.stream_contexts.get(stream_id) + if ctx: + ctx.update_result(result_text, start_sec, end_sec) + print( + f"{GST_PLUGIN_NAME}{stream_label}: Completed " + f"(total: {ctx.segments_completed})" + ) + # Print full result with timestamp prefix + print( + f"{GST_PLUGIN_NAME}{stream_label}: Result: " + f"{start_sec:.2f}s-{end_sec:.2f}s {result_text}" + ) + + # Emit signal with result + self.emit( + "vlm-result", + stream_id, + start_sec, + end_sec, + result_text, + ) + else: + print( + f"{GST_PLUGIN_NAME}{stream_label}: " + f"VLM returned empty result" + ) + except Exception as e: + msg = f"{GST_PLUGIN_NAME}{stream_label}: Worker error - {e}" + print(msg) + import traceback + + traceback.print_exc() + finally: + self._infer_queue.task_done() + + print(f"{GST_PLUGIN_NAME}: Inference worker stopped") + + def _run_vlm_batch( + self, segment: Segment, prompt_config: Dict + ) -> Optional[str]: # noqa: BLK100 + """Run VLM inference on a segment""" + if self.llm is None: + return None + + try: + # Collect frame tensors and timestamps + frame_tensors = [] + frame_times = [] + + for i, frame_data in enumerate(segment.frames): + tensor = frame_data.tensor_gpu + if tensor.dim() == 3: + tensor = tensor.permute(2, 0, 1) + frame_tensors.append(tensor) + frame_time_sec = frame_data.pts / 1_000_000_000.0 + frame_times.append(frame_time_sec) + + if not frame_tensors: + return None + + # Stack into batch + batch_tensor = torch.stack(frame_tensors) + + # Calculate FPS + if len(frame_times) > 1: + time_diff = frame_times[-1] - frame_times[0] + num_intervals = len(frame_times) - 1 + fps = num_intervals / time_diff if time_diff > 0 else 1.0 + else: + fps = 1.0 + + # Video metadata + if len(frame_times) > 1: + duration = frame_times[-1] - frame_times[0] + else: + duration = 0.0 + video_metadata = { + "total_num_frames": len(frame_tensors), + "frames_indices": [int(t * fps) for t in frame_times], + "fps": fps, + "duration": duration, + } + + # Build timestamp string + string_of_times = " ".join([f"{t:.2f}s" for t in frame_times]) + num_frames = len(frame_tensors) + + # Build SamplingParams with required parameters + sampling_params_dict = { + "temperature": prompt_config.get("temperature", 0.2), + "max_tokens": prompt_config.get("max_tokens", 64), + } + + # Add optional parameters if specified + if "top_p" in prompt_config and prompt_config["top_p"] is not None: + sampling_params_dict["top_p"] = prompt_config["top_p"] + + if "top_k" in prompt_config and prompt_config["top_k"] is not None: + sampling_params_dict["top_k"] = prompt_config["top_k"] + + if ( + "repetition_penalty" in prompt_config + and prompt_config["repetition_penalty"] is not None + ): + sampling_params_dict["repetition_penalty"] = prompt_config[ + "repetition_penalty" + ] + + sampling_params = SamplingParams(**sampling_params_dict) + + # Use chat template + has_chat_template = hasattr(self.tokenizer, "apply_chat_template") + if self.tokenizer and has_chat_template: + # Determine mode: video_mode=1 uses video input (all frames), + # video_mode=0 uses multi-image input + if self.video_mode == 0: + # Image mode: pass all frames as separate images + # Get user prompt with default + user_prompt = prompt_config.get( + "user_prompt", + "These are {num_frames} images from stream " + "{stream_id} sampled at timestamps {timestamps}. " + "Describe the scene in detail.", + ) + + # Format prompt with placeholders + prompt_text = self._format_user_prompt( + user_prompt, + segment.stream_id, + num_frames, + string_of_times, + ) + + # Build content with text + multiple images + content = [{"type": "text", "text": prompt_text}] + for i in range(num_frames): + img_entry = {"type": "image", "image": f"frame{i}.jpg"} + content.append(img_entry) + + # Build messages - only include system prompt if not None + system_prompt = prompt_config.get( + "system_prompt", self._system_prompt + ) + if system_prompt is not None: + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": content}, + ] + else: + messages = [ + {"role": "user", "content": content}, + ] + + prompt_text = self.tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + + # Convert frames to target format (pytorch/pil/numpy) + image_tensors = [ + self._convert_tensor_to_format( + batch_tensor[i], self.tensor_format + ) + for i in range(num_frames) + ] + inputs = { + "prompt": prompt_text, + "multi_modal_data": {"image": image_tensors}, + } + + elif num_frames == 1: + # Single frame in video_mode=1: use image input. + # Video processors (e.g. Qwen3VL) require >=2 frames; + # passing as image avoids that constraint. + user_prompt = prompt_config.get( + "user_prompt", + "This is an image from stream {stream_id} at " + "timestamp {timestamps}. Describe the scene.", + ) + + prompt_text = self._format_user_prompt( + user_prompt, + segment.stream_id, + num_frames, + string_of_times, + ) + + system_prompt = prompt_config.get( + "system_prompt", self._system_prompt + ) + if system_prompt is not None: + messages = [ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": [ + {"type": "text", "text": prompt_text}, + {"type": "image", "image": "frame.jpg"}, + ], + }, + ] + else: + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt_text}, + {"type": "image", "image": "frame.jpg"}, + ], + }, + ] + + prompt_text = self.tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + + # Convert [C, H, W] tensor to [H, W, C] numpy array. + # vLLM interprets a torch.Tensor in image mm_data as + # pre-computed image_embeds; numpy avoids that error. + image_np = ( + batch_tensor[0].cpu().permute(1, 2, 0).numpy() + ) + inputs = { + "prompt": prompt_text, + "multi_modal_data": {"image": image_np}, + } + + else: + # Video mode - multiple frames with video metadata + # Get user prompt with default + user_prompt = prompt_config.get( + "user_prompt", + "This is a video from stream {stream_id} with " + "{num_frames} frames sampled at timestamps " + "{timestamps}. Describe the video content.", + ) + + # Format prompt with placeholders + prompt_text = self._format_user_prompt( + user_prompt, + segment.stream_id, + num_frames, + string_of_times, + ) + + # Build messages - only include system prompt if not None + system_prompt = prompt_config.get( + "system_prompt", self._system_prompt + ) + if system_prompt is not None: + messages = [ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": [ + {"type": "text", "text": prompt_text}, + {"type": "video", "video": "segment.mp4"}, + ], + }, + ] + else: + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt_text}, + {"type": "video", "video": "segment.mp4"}, + ], + }, + ] + + prompt_text = self.tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + video_input = (batch_tensor.cpu(), video_metadata) + inputs = { + "prompt": prompt_text, + "multi_modal_data": {"video": video_input}, + } + + outputs = self.llm.generate( + inputs, sampling_params=sampling_params + ) # noqa: BLK100 + if outputs: + return outputs[0].outputs[0].text + + return None + + except Exception as e: + msg = f"{GST_PLUGIN_NAME}: VLM inference failed - {e}" + print(msg) + import traceback + + traceback.print_exc() + return None + + def do_stop(self) -> bool: + """Stop processing and finalize all streams""" + n = len(self.stream_contexts) + lbl = "stream" if n == 1 else "stream(s)" + print(f"{GST_PLUGIN_NAME}: Stopping - finalizing {n} {lbl}") + + # Finalize remaining segments for each stream + with self.stream_contexts_lock: + for stream_id, ctx in self.stream_contexts.items(): + ctx_label = f"[Stream {stream_id}]" if n > 1 else "" + + if ctx.open_segments: + print( + f"{GST_PLUGIN_NAME}{ctx_label}: Finalizing " + f"{len(ctx.open_segments)} remaining segments" + ) + for seg in ctx.open_segments[:]: + if seg.frames: + start_sec = seg.start_pts_ns / 1_000_000_000 + end_sec = seg.end_pts_ns / 1_000_000_000 + pts_diff = seg.frames[-1].pts - seg.frames[0].pts + if len(seg.frames) > 1: + duration_sec = pts_diff / 1_000_000_000 + else: + duration_sec = 0 + + msg = ( + f"{GST_PLUGIN_NAME}{ctx_label}: Submitting " + f"incomplete segment " + f"[{start_sec:.2f}s - {end_sec:.2f}s] " + f"with {len(seg.frames)} frames " + f"(duration: {duration_sec:.2f}s)" + ) + print(msg) + + prompt_config = { + "user_prompt": self.user_prompt, + "system_prompt": self._system_prompt, + "max_tokens": self.max_tokens, + "temperature": self.temperature, + } + + # Add optional sampling parameters if specified + if self.top_p is not None: + prompt_config["top_p"] = self.top_p + if self.top_k is not None: + prompt_config["top_k"] = self.top_k + if self.repetition_penalty is not None: + prompt_config["repetition_penalty"] = ( + self.repetition_penalty + ) + + request = SegmentRequest( + stream_id, seg, prompt_config + ) # noqa: BLK100 + try: + self._infer_queue.put_nowait(request) + ctx.segments_submitted += 1 + ctx.total_frames_in_segments += len(seg.frames) + except Exception: + ctx.segments_dropped += 1 + + # Wait for queue to drain AND all processing to complete + import time + + max_wait = self.max_wait_timeout + elapsed = 0.0 + + print(f"{GST_PLUGIN_NAME}: Waiting for all segments to complete...") + + while elapsed < max_wait: + # Check both queue and per-stream completion status + queue_empty = self._infer_queue.empty() + + all_complete = True + pending_count = 0 + with self.stream_contexts_lock: + for ctx in self.stream_contexts.values(): + pending = ( + ctx.segments_submitted + - ctx.segments_completed + - ctx.segments_dropped + ) + if pending > 0: + all_complete = False + pending_count += pending + + if queue_empty and all_complete: + print( + f"{GST_PLUGIN_NAME}: All segments completed " + f"successfully (waited {elapsed:.1f}s)" + ) + break + + time.sleep(0.5) + elapsed += 0.5 + + if int(elapsed) % 10 == 0 and pending_count > 0: + print( + f"{GST_PLUGIN_NAME}: Still processing... " + f"{pending_count} segment(s) remaining " + f"({elapsed:.0f}s elapsed)" + ) + + if not all_complete: + print( + f"{GST_PLUGIN_NAME}: WARNING: Timeout waiting for " + f"segments. Some segments may not be fully processed." + ) + + # Give a small grace period to ensure any final result updates + # are complete + time.sleep(0.5) + + # Stop worker thread + print(f"\n{GST_PLUGIN_NAME}: Stopping inference worker...") + self._stop_event.set() + if self._infer_thread: + self._infer_thread.join(timeout=5.0) + self._infer_thread = None + + # Shut down vLLM engine to release GPU memory + if self.llm is not None: + print(f"{GST_PLUGIN_NAME}: Shutting down vLLM engine...") + try: + if hasattr(self.llm, "shutdown"): + self.llm.shutdown() + elif hasattr(self.llm, "llm_engine") and hasattr( + self.llm.llm_engine, "shutdown" + ): + self.llm.llm_engine.shutdown() + except Exception as e: + print( + f"{GST_PLUGIN_NAME}: Warning: vLLM shutdown error: {e}" + ) + finally: + self.llm = None + + # Print statistics for each stream + # (AFTER worker stops to ensure final counts) + num_streams = len(self.stream_contexts) + if num_streams == 1: + stats_label = "Statistics" + else: + stats_label = "Multi-Stream Statistics" + print(f"\n{GST_PLUGIN_NAME}: Final {stats_label}:") + + with self.stream_contexts_lock: + for stream_id, ctx in self.stream_contexts.items(): + # For single stream, omit "Stream 0:" for cleaner output + if num_streams == 1: + print(f" Frames processed: {ctx.frame_counter}") + print(f" Segments submitted: {ctx.segments_submitted}") + print(f" Segments completed: {ctx.segments_completed}") + print(f" Segments dropped: {ctx.segments_dropped}") + total_frames = ctx.total_frames_in_segments + print(f" Total frames in segments: {total_frames}") + if ctx.segments_submitted > 0: + n_frames = ctx.total_frames_in_segments + n_segs = ctx.segments_submitted + avg_frames = n_frames / n_segs + print(f" Avg frames per segment: {avg_frames:.1f}") + else: + print(f"\n Stream {stream_id}:") + print(f" Frames processed: {ctx.frame_counter}") + print(f" Segments submitted: {ctx.segments_submitted}") + print(f" Segments completed: {ctx.segments_completed}") + print(f" Segments dropped: {ctx.segments_dropped}") + total_frames = ctx.total_frames_in_segments + print(f" Total frames in segments: {total_frames}") + if ctx.segments_submitted > 0: + n_frames = ctx.total_frames_in_segments + n_segs = ctx.segments_submitted + avg_frames = n_frames / n_segs + print(f" Avg frames per segment: {avg_frames:.1f}") + + print(f"{GST_PLUGIN_NAME}: Shutdown complete") + return True + + +GObject.type_register(NvVllmVLM) +__gstelementfactory__ = (GST_PLUGIN_NAME, Gst.Rank.NONE, NvVllmVLM) diff --git a/deepstream-vllm-plugin/install.sh b/deepstream-vllm-plugin/install.sh new file mode 100755 index 0000000..7ae7754 --- /dev/null +++ b/deepstream-vllm-plugin/install.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Script to set up environment for VLLM DS plugin + +set -e # Exit immediately if a command exits with a non-zero status + +# 1) Install Python dependencies from requirements.txt +# --ignore-installed forces reinstall, useful if system packages conflict +pip install -r requirements.txt --ignore-installed + +# 2) Update apt package index to get latest package metadata +apt update + +# 3) Install Python GObject Introspection bindings (used with GStreamer) +apt install -y python3-gi + +# 4) Install GStreamer Python 3 bindings +apt install -y python3-gst-1.0 + +# 5) Install the GStreamer Python plugin loader +apt install -y gstreamer1.0-python3-plugin-loader + diff --git a/deepstream-vllm-plugin/requirements.txt b/deepstream-vllm-plugin/requirements.txt new file mode 100644 index 0000000..4adf55c --- /dev/null +++ b/deepstream-vllm-plugin/requirements.txt @@ -0,0 +1,5 @@ +vllm==0.15.1 +torch==2.9.1 +kafka-python +PyYAML +Pillow diff --git a/deepstream-vllm-plugin/test_consumer.py b/deepstream-vllm-plugin/test_consumer.py new file mode 100644 index 0000000..2355c00 --- /dev/null +++ b/deepstream-vllm-plugin/test_consumer.py @@ -0,0 +1,229 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### + +""" +Kafka consumer for VLM results. +Continuously listens for VLM inference results and prints them in real-time. +""" + +import argparse +import json +import sys +import uuid +from datetime import datetime + +try: + from kafka import KafkaConsumer +except ImportError: + print("Error: kafka-python not installed") + print("Install with: pip install kafka-python") + sys.exit(1) + + +def main(): + # Parse arguments + parser = argparse.ArgumentParser( + description="Kafka consumer for VLM results", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Listen for messages with 5-minute timeout (default behavior) + python3 test_consumer.py + + # Specify custom Kafka broker and topic + python3 test_consumer.py --broker localhost:9092 --topic vlm-result + + # Read all existing messages from beginning with a fresh consumer group + python3 test_consumer.py --reset --topic vlm-result + + # Keep running forever until Ctrl+C (no timeout) + python3 test_consumer.py --timeout 0 + + # Exit after 30 seconds of no new messages + python3 test_consumer.py --timeout 30000 + + # Start from latest (only read new messages going forward) + python3 test_consumer.py --from-latest + """, + ) + parser.add_argument( + "--broker", + default="localhost:9092", + help="Kafka bootstrap servers (default: localhost:9092)", + ) + parser.add_argument( + "--topic", + default="vlm-results", + help="Kafka topic name (default: vlm-results)", + ) + parser.add_argument( + "--reset", + action="store_true", + help="Use unique consumer group to read all messages from beginning", + ) + parser.add_argument( + "--timeout", + type=int, + default=300000, + help=( + "Exit after N milliseconds of no messages " + "(default: 300000 = 5 minutes, use 0 for no timeout)" + ), + ) + parser.add_argument( + "--from-latest", + action="store_true", + help="Start from latest messages instead of earliest", + ) + + args = parser.parse_args() + + bootstrap_servers = args.broker + topic = args.topic + + # Consumer group + if args.reset: + group_id = f"vlm-consumer-{uuid.uuid4().hex[:8]}" + print( + "🔄 RESET MODE: Using unique consumer group " # noqa: BLK100 + "(will read from beginning)" + ) + else: + group_id = "vlm-consumer-group" + + # Offset reset + offset_reset = "latest" if args.from_latest else "earliest" + + # Handle timeout (0 means no timeout) + timeout_ms = None if args.timeout == 0 else args.timeout + + print("Starting Kafka consumer...") + print(f" Bootstrap servers: {bootstrap_servers}") + print(f" Topic: {topic}") + print(f" Consumer group: {group_id}") + print(f" Reading from: {offset_reset}") + if timeout_ms: + timeout_mins = timeout_ms / 60000 + print( + f" Timeout: {timeout_ms}ms ({timeout_mins:.1f} minutes - " + "will exit if no messages)" + ) + else: + print(" Mode: Continuous (will keep running until Ctrl+C)") + print("=" * 80) + + try: + consumer = KafkaConsumer( + topic, + bootstrap_servers=bootstrap_servers, + value_deserializer=lambda m: json.loads(m.decode("utf-8")), + auto_offset_reset=offset_reset, + enable_auto_commit=True, + group_id=group_id, + consumer_timeout_ms=timeout_ms, # None = run forever + ) + except Exception as e: + print(f"\n✗ Failed to connect to Kafka: {e}") + print("\nTroubleshooting:") + print(" 1. Ensure Kafka is running: docker ps | grep kafka") + print(" 2. Check connection: telnet localhost 9092") + print( # noqa: BLK100 + " 3. Start Kafka: " # noqa: BLK100 + "docker-compose -f docker-compose-kafka.yml up -d" + ) + sys.exit(1) + + print("\n✓ Connected successfully!") + print("Listening for messages... (Press Ctrl+C to stop)\n") + print("=" * 80) + + message_count = 0 + start_time = datetime.now() + + try: + for message in consumer: + message_count += 1 + data = message.value + + timestamp = datetime.now().strftime('%H:%M:%S') # noqa: BLK100 + print(f"\n📥 Message #{message_count} [{timestamp}]") + print("─" * 80) + print(" Kafka Metadata:") + print(f" Partition: {message.partition}") + print(f" Offset: {message.offset}") + key_str = ( + message.key.decode('utf-8') if message.key else 'None' + ) + print(f" Key: {key_str}") + msg_time = datetime.fromtimestamp( + message.timestamp / 1000 + ).strftime('%Y-%m-%d %H:%M:%S') + print(f" Timestamp: {msg_time}") + print("\n VLM Result:") + print(f" Stream ID: {data['stream_id']}") + start_t = data['segment']['start_time'] + end_t = data['segment']['end_time'] + print(f" Time Range: {start_t:.1f}s - {end_t:.1f}s") + print(f" Duration: {data['segment']['duration']:.1f}s") + + # Format result text with wrapping + result_text = data["result"] + if len(result_text) > 200: + # Wrap long text + print(" Result:") + for i in range(0, len(result_text), 100): + print(f" {result_text[i:i+100]}") + else: + print(f" Result: {result_text}") + + print("\n Metadata:") + print(f" Source: {data['metadata']['source']}") + print(f" Version: {data['metadata']['version']}") + print(f" Publish Time: {data.get('timestamp', 'N/A')}") + print("─" * 80) + + # If we reach here naturally (not via exception), it's a timeout + if timeout_ms: + timeout_secs = timeout_ms / 1000 + print( + f"\n⏱️ Timeout reached " + f"({timeout_secs:.0f}s of no new messages)" + ) + + except KeyboardInterrupt: + print("\n\n⚠️ Interrupted by user (Ctrl+C)") + except Exception as e: + print(f"\n✗ Error consuming messages: {e}") + import traceback + + traceback.print_exc() + + consumer.close() + + elapsed = (datetime.now() - start_time).total_seconds() + print("\n" + "=" * 80) + print("Session Summary") + print("=" * 80) + print(f" Messages consumed: {message_count}") + print(f" Duration: {elapsed:.1f}s") + if message_count > 0 and elapsed > 0: + print(f" Rate: {message_count / elapsed:.2f} messages/sec") + print("=" * 80) + + +if __name__ == "__main__": + main() diff --git a/deepstream-vllm-plugin/vllm_ds_app_kafka_publish.py b/deepstream-vllm-plugin/vllm_ds_app_kafka_publish.py new file mode 100644 index 0000000..e3d1494 --- /dev/null +++ b/deepstream-vllm-plugin/vllm_ds_app_kafka_publish.py @@ -0,0 +1,500 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### + +""" +DeepStream VLM application with Kafka publishing and signal-based result +handling. +Supports both single-stream and multi-stream processing with file and RTSP +sources. + +Features: +- Single-stream and multi-stream VLM processing +- Real-time result delivery via GObject signals +- Kafka topic publishing for downstream processing +- Dry-run mode for testing without Kafka +- Efficient event-driven architecture +- File and RTSP source support via uridecodebin +""" + +import json +import os +import sys +import time +from typing import Optional + +import gi +gi.require_version("Gst", "1.0") # noqa: E402, I003, BLK100 +from gi.repository import GLib, Gst # noqa: E402, I003 + +# Register the custom plugin +import gstnvvllmvlm # noqa: E402 + +Gst.Element.register(None, "nvvllmvlm", Gst.Rank.NONE, gstnvvllmvlm.NvVllmVLM) + +# Kafka imports (with graceful fallback) +try: + from kafka import KafkaProducer + from kafka.errors import KafkaError + + KAFKA_AVAILABLE = True +except ImportError: + KAFKA_AVAILABLE = False + print("Warning: kafka-python not installed. Run: pip install kafka-python") + + +def to_uri(path_or_uri: str) -> str: + """Convert a file path or URI string to a GStreamer-compatible URI.""" + if "://" in path_or_uri: + return path_or_uri + return "file://" + os.path.abspath(path_or_uri) + + +class VLMKafkaSignalPublisher: + """ + Kafka publisher that uses GObject signals to receive VLM results. + More efficient than polling - publishes immediately when results are + available. + """ + + def __init__(self, kafka_config: dict, topic: str, dry_run: bool = False): + """ + Initialize Kafka publisher. + + Args: + kafka_config: Kafka connection configuration + topic: Topic name to publish to + dry_run: If True, print messages instead of sending to Kafka + """ + self.topic = topic + self.dry_run = dry_run + self.producer: Optional[KafkaProducer] = None + self.messages_sent = 0 + self.messages_failed = 0 + + # Initialize Kafka producer + if not dry_run and KAFKA_AVAILABLE: + try: + self.producer = KafkaProducer( + bootstrap_servers=kafka_config.get( + "bootstrap_servers", "localhost:9092" + ), + value_serializer=lambda v: json.dumps(v).encode("utf-8"), + key_serializer=lambda k: k.encode("utf-8") if k else None, + acks="all", + retries=3, + # Required for idempotence + max_in_flight_requests_per_connection=1, + enable_idempotence=True, + compression_type="gzip", + linger_ms=100, + batch_size=16384, + ) + print(f"✓ Kafka producer initialized (topic: {self.topic})") + except Exception as e: + print(f"✗ Failed to initialize Kafka producer: {e}") + print(" Falling back to dry-run mode (console output only)") + self.dry_run = True + self.producer = None + print("✓ Dry-run mode enabled") + else: + if not KAFKA_AVAILABLE: + print("✗ Kafka not available - dry-run mode enabled") + else: + print("✓ Dry-run mode enabled (console output only)") + self.producer = None + + def on_vlm_result( + self, element, stream_id, start_time, end_time, result_text + ): + """ + Signal handler for vlm-result signal. + Called immediately when VLM inference completes. + + Args: + element: The nvvllmvlm element that emitted the signal + stream_id: Stream identifier + start_time: Segment start time in seconds + end_time: Segment end time in seconds + result_text: VLM inference result + """ + # Construct message + message = { + "stream_id": stream_id, + "timestamp": time.time(), + "segment": { + "start_time": start_time, + "end_time": end_time, + "duration": end_time - start_time, + }, + "result": result_text, + "metadata": {"source": "vllm-ds-plugin", "version": "1.0"}, + } + + # Publish to Kafka or print to console + self.publish(message, stream_id) + + def publish(self, message: dict, stream_id: int): + """ + Publish message to Kafka or print to console. + + Args: + message: Message payload + stream_id: Stream ID (used as partition key) + """ + # Use stream_id as partition key for ordering + partition_key = f"stream_{stream_id}" + + if self.dry_run or self.producer is None: + # Dry-run mode: print to console + print(f"\n{'='*80}") + print("📤 KAFKA MESSAGE (Dry-Run)") + print(f"{'='*80}") + print(f"Topic: {self.topic}") + print(f"Key: {partition_key}") + print(f"Value: {json.dumps(message, indent=2)}") + print(f"{'='*80}\n") + self.messages_sent += 1 + else: + # Send to Kafka + try: + future = self.producer.send( + self.topic, key=partition_key, value=message + ) + + # Optional: wait for acknowledgment + record_metadata = future.get(timeout=10) + + self.messages_sent += 1 + print( + f"✓ Published to Kafka: stream={stream_id}, " + f"time={message['segment']['start_time']:.1f}s-" + f"{message['segment']['end_time']:.1f}s, " + f"partition={record_metadata.partition}, " + f"offset={record_metadata.offset}" + ) + + except KafkaError as e: + self.messages_failed += 1 + print(f"✗ Kafka publish failed: {e}") + except Exception as e: + self.messages_failed += 1 + print(f"✗ Unexpected error during publish: {e}") + + def close(self): + """Close Kafka producer and print statistics""" + if self.producer: + print("\nFlushing Kafka producer...") + self.producer.flush(timeout=10) + self.producer.close() + + print(f"\n{'='*80}") + print("KAFKA PUBLISHER STATISTICS") + print(f"{'='*80}") + print(f"Messages sent: {self.messages_sent}") + print(f"Messages failed: {self.messages_failed}") + print(f"{'='*80}\n") + + +class VLMKafkaApp: + """DeepStream VLM app with Kafka publishing via signals + (single or multi-stream, file or RTSP sources)""" + + def __init__(self, input_uris, kafka_config, topic, dry_run=False): + """ + Initialize application. + + Args: + input_uris: List of GStreamer-compatible URIs (file:// or rtsp://) + kafka_config: Kafka connection configuration + topic: Kafka topic name + dry_run: If True, print messages instead of sending to Kafka + """ + self.input_uris = input_uris + self.num_sources = len(input_uris) + self.pipeline = None + self.loop = None + self.streams_eos = set() + + # Initialize Kafka publisher + self.kafka_publisher = VLMKafkaSignalPublisher( + kafka_config, topic, dry_run + ) + + def bus_call(self, bus, message, loop): + """Handle GStreamer bus messages""" + t = message.type + + if t == Gst.MessageType.EOS: + print("End-of-stream") + loop.quit() + elif t == Gst.MessageType.WARNING: + err, debug = message.parse_warning() + print(f"Warning: {err}: {debug}") + elif t == Gst.MessageType.ERROR: + err, debug = message.parse_error() + print(f"Error: {err}: {debug}") + loop.quit() + + return True + + def pad_probe_callback(self, pad, info, stream_id): + """Probe to detect per-stream EOS""" + gst_buffer = info.get_buffer() + if gst_buffer: + if gst_buffer.pts == Gst.CLOCK_TIME_NONE: + print(f"Stream {stream_id}: Received EOS") + self.streams_eos.add(stream_id) + + if len(self.streams_eos) == self.num_sources: + print(f"All {self.num_sources} stream(s) finished") + + return Gst.PadProbeReturn.OK + + def build_pipeline(self): + """Build the GStreamer pipeline. + + Uses uridecodebin per stream so that both file:// and rtsp:// URIs + are supported transparently. uridecodebin selects the appropriate + source plugin, demuxer, parser, and hardware decoder automatically. + """ + print(f"Building pipeline for {self.num_sources} source(s)...") + + has_live = any( + uri.startswith("rtsp://") or uri.startswith("rtsps://") + for uri in self.input_uris + ) + + # Create pipeline + self.pipeline = Gst.Pipeline.new("vlm-kafka-signal-pipeline") + + # Create streammux + streammux = Gst.ElementFactory.make("nvstreammux", "stream-muxer") + streammux.set_property("width", 1920) + streammux.set_property("height", 1080) + streammux.set_property("batch-size", self.num_sources) + streammux.set_property("live-source", has_live) + if not has_live: + streammux.set_property("batched-push-timeout", 4000000) + + # Add to pipeline + self.pipeline.add(streammux) + + # Pre-request mux sink pads so pad-added callbacks can link into them + mux_sink_pads = [] + for i in range(self.num_sources): + sink_pad = streammux.request_pad_simple(f"sink_{i}") + if not sink_pad: + print(f"Error: Could not get sink pad {i} from streammux") + return None + sink_pad.add_probe( + Gst.PadProbeType.BUFFER, self.pad_probe_callback, i + ) + mux_sink_pads.append(sink_pad) + + # Create one uridecodebin per source + for i, uri in enumerate(self.input_uris): + print(f" Source {i}: {uri}") + + uri_decode_bin = Gst.ElementFactory.make( + "uridecodebin", f"uri-decode-bin-{i}" + ) + if not uri_decode_bin: + print( + f"Error: Could not create uridecodebin for stream {i}" + ) + return None + + uri_decode_bin.set_property("uri", uri) + self.pipeline.add(uri_decode_bin) + + # Capture loop variables via default args + def on_pad_added( + element, + pad, + mux_sinkpad=mux_sink_pads[i], + stream_id=i, + ): + caps = pad.get_current_caps() + if not caps: + caps = pad.query_caps() + if not caps: + return + structure = caps.get_structure(0) + if "video" in structure.get_name(): + if not mux_sinkpad.is_linked(): + if pad.link(mux_sinkpad) == Gst.PadLinkReturn.OK: + print( + f" Linked uridecodebin → " + f"streammux.sink_{stream_id}" + ) + + uri_decode_bin.connect("pad-added", on_pad_added) + + # Video converter + nvvidconv = Gst.ElementFactory.make("nvvideoconvert", "convertor") + nvvidconv.set_property("nvbuf-memory-type", 0) + + # Caps filter for RGB + caps_filter = Gst.ElementFactory.make("capsfilter", "caps-filter") + caps_rgb = Gst.Caps.from_string("video/x-raw(memory:NVMM), format=RGB") + caps_filter.set_property("caps", caps_rgb) + + # VLM plugin - uses configuration from config.yaml + nvvllm = Gst.ElementFactory.make("nvvllmvlm", "vlm-infer") + + # Connect signal to Kafka publisher + nvvllm.connect("vlm-result", self.kafka_publisher.on_vlm_result) + print("✓ Connected vlm-result signal to Kafka publisher") + + # Fakesink + sink = Gst.ElementFactory.make("fakesink", "fake-sink") + sink.set_property("sync", False) + + # Add elements to pipeline + self.pipeline.add(nvvidconv) + self.pipeline.add(caps_filter) + self.pipeline.add(nvvllm) + self.pipeline.add(sink) + + # Link pipeline + streammux.link(nvvidconv) + nvvidconv.link(caps_filter) + caps_filter.link(nvvllm) + nvvllm.link(sink) + + print("Pipeline built successfully\n") + + return self.pipeline + + def run(self): + """Run the application""" + # Build pipeline + pipeline = self.build_pipeline() + + # Set up bus + bus = pipeline.get_bus() + bus.add_signal_watch() + + # Create main loop + self.loop = GLib.MainLoop() + bus.connect("message", self.bus_call, self.loop) + + # Start pipeline + print("Starting pipeline...") + pipeline.set_state(Gst.State.PLAYING) + + try: + print("Running... (Press Ctrl+C to stop)\n") + self.loop.run() + except KeyboardInterrupt: + print("\nInterrupted by user") + + # Cleanup + print("\nStopping pipeline...") + pipeline.set_state(Gst.State.NULL) + + # Close Kafka publisher + self.kafka_publisher.close() + + +def main(): + """Main entry point""" + import argparse + + parser = argparse.ArgumentParser( + description="DeepStream VLM app with Kafka publishing " + "(single-stream or multi-stream, file or RTSP sources)", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +URIs can be: + File paths: /path/to/video.mp4 (auto-converted to file:// URI) + File URIs: file:///path/to/video.mp4 + RTSP streams: rtsp://user:pass@host:port/stream + +Examples: + # Single file with dry-run (console output) + python3 vllm_ds_app_kafka_publish.py video1.mp4 --dry-run + + # RTSP stream with dry-run + python3 vllm_ds_app_kafka_publish.py rtsp://192.168.1.100:8554/stream \\ + --dry-run + + # Single file with Kafka publishing + python3 vllm_ds_app_kafka_publish.py video1.mp4 \\ + --kafka-bootstrap localhost:9092 \\ + --topic vlm-results + + # Multi-stream with mixed sources and Kafka + python3 vllm_ds_app_kafka_publish.py \\ + video1.mp4 rtsp://192.168.1.100:8554/stream \\ + --kafka-bootstrap localhost:9092 \\ + --topic vlm-results + """, + ) + + parser.add_argument( + "sources", + nargs="+", + help="Video file paths or URIs to process (file paths, file://, rtsp://)", + ) + parser.add_argument( + "--kafka-bootstrap", + default="localhost:9092", + help="Kafka bootstrap servers (default: localhost:9092)", + ) + parser.add_argument( + "--topic", + default="vlm-results", + help="Kafka topic name (default: vlm-results)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print messages to console instead of sending to Kafka", + ) + + args = parser.parse_args() + + # Initialize GStreamer + Gst.init(None) + + # Convert bare file paths to file:// URIs; validate files exist + input_uris = [] + for src in args.sources: + uri = to_uri(src) + if uri.startswith("file://"): + file_path = uri[len("file://"):] + if not os.path.exists(file_path): + print(f"Error: File not found: {file_path}") + sys.exit(1) + input_uris.append(uri) + + # Kafka configuration + kafka_config = {"bootstrap_servers": args.kafka_bootstrap} + + # Create and run app + app = VLMKafkaApp( + input_uris=input_uris, + kafka_config=kafka_config, + topic=args.topic, + dry_run=args.dry_run, + ) + app.run() + + +if __name__ == "__main__": + main() diff --git a/deepstream_parallel_inference_app/README.md b/deepstream_parallel_inference_app/README.md new file mode 100755 index 0000000..9120417 --- /dev/null +++ b/deepstream_parallel_inference_app/README.md @@ -0,0 +1,230 @@ +# Parallel Multiple Models App +## Introduction +The parallel inferencing application constructs the parallel inferencing branches pipeline as the following graph, so that the multiple models can run in parallel in one piepline. + +![Pipeline_Diagram](common.png) + +## Main Features + +* Support multiple models inference with [nvinfer](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvinfer.html)(TensorRT) or [nvinferserver](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvinferserver.html)(Triton) in parallel +* Support sources selection for different models with [nvstreammux](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvstreammux.html) and [nvstreamdemux](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvstreamdemux.html) +* Support [new nvstreammux](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvstreammux2.html). you can set "export USE_NEW_NVSTREAMMUX=yes" to use new streammux. +* Support to mux output meta from different sources and different models with **gst-nvdsmetamux** plugin newly introduced in DeepStream 6.1.1 or above +* Support latency measurement and frame rate measurement with the enviroment varialble enabling. + + The latency measurement can be enabled by set the following enviroment varialbles: + ``` + export NVDS_ENABLE_COMPONENT_LATENCY_MEASUREMENT=1 + export NVDS_ENABLE_LATENCY_MEASUREMENT=1 + ``` + + +# Prerequisites +- If you are using a deepstream docker above DeepStream 6.1.1 version, please execute /opt/nvidia/deepstream/deepstream/user_additional_install.sh to install tools such as x264enc which is used in this sample. +- DeepStream 6.1.1 or above, especially + - [nvmsgbroker](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvmsgbroker.html) if you want to enable nvmsgbroker sink, e.g. [Kafka](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvmsgbroker.html#nvds-kafka-proto-kafka-protocol-adapter) + - [nvinferserver](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvinferserver.html) if running model with Triton +- Cloud server, e.g. Kafka server (version >= kafka_2.12-3.2.0), if you want to enable broker sink + + + +# Steps To Run + +The sample should be downloaded and built with **root** permission. + +1. Download + + ``` + apt install git-lfs + git lfs install --skip-repo + cd deepstream_parallel_inference_app + git lfs pull + ``` + + If git-lfs download fails for bodypose2d and YoloV4 models, get them from download [link](https://nvidia.box.com/s/rqb5t5koigvpmobkxj1245gv09d9u2sx) + +2. Generate Inference Engines + + Below instructions are only needed on **Jetson** ([Jetpack 5.0.2 or above](https://developer.nvidia.com/embedded/jetpack-sdk-502)) + + ``` + apt-get install -y libjson-glib-dev libgstrtspserver-1.0-dev + /opt/nvidia/deepstream/deepstream/samples/triton_backend_setup.sh + ## Only DeepStream 6.1.1 GA need to copy the metamux plugin library. Skip this copy command if DeepStream version is above 6.1.1 GA + cp tritonclient/sample/gst-plugins/gst-nvdsmetamux/libnvdsgst_metamux.so /opt/nvidia/deepstream/deepstream/lib/gst-plugins/libnvdsgst_metamux.so + ## set power model and boost CPU/GPU/EMC clocks + nvpmodel -m 0 && jetson_clocks + ``` + + Below instructions are needed for both **Jetson** and **dGPU** (DeepStream Triton docker - [6.1.1-triton or above](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/deepstream/tags)) + + ``` + cd tritonserver/ + ./build_engine.sh + ``` + +3. Build and Run + + ``` + cd tritonclient/sample/ + source build.sh + ./apps/deepstream-parallel-infer/deepstream-parallel-infer -c configs/apps/bodypose_yolo_lpr/source4_1080p_dec_parallel_infer.yml + ``` + + + +# Directory + +![Files](files.PNG) + + + +# Application Configuration Semantics + +The parallel inferencing app uses the YAML configuration file to config GIEs, sources, and other features of the pipeline. The basic group semantics is the same as [deepstream-app](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_ref_app_deepstream.html#expected-output-for-the-deepstream-reference-application-deepstream-app). + +Please refer to deepstream-app [Configuration Groups](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_ref_app_deepstream.html#configuration-groups) part for the semantics of corresponding groups. + +There are additional new groups introduced by the parallel inferencing app which enable the app to select sources for different inferencing branches and to select output metadata for different inferencing GIEs: + +### Branch Group +The branch group specifies the sources to be infered by the specific inferencing branch. The selected sources are identified by the source IDs list. The inferencing branch is identified by the first PGIE unique-id in this branch. For example: +``` +branch0: + key1: value1 + key2: value2 +``` + +The branch group properties are: + +| Key | Meaning | Type and Value | Example | Plateforms | +|-------|-------------------------------------------------------|----------------------------------|----------------|------------| +|pgie-id|the first PGIE unique-id in this branch | Integer, >0 |pgie-id: 8 |dGPU, Jetson| +|src-ids|The source-id list of selected sources for this branch | Semicolon separated integer array|src-ids: 0;2;5;6|dGPU, Jetson| + +### Metamux Group + +The metamux group specifies the configuration file of gst-dsmetamux plugin. For example: +``` +meta-mux: + key1: value1 + key2: value2 +``` +The metamux group properties are: + +| Key | Meaning | Type and Value | Example | Plateforms | +|-----------|------------------------------------------------------------|----------------|---------------------------------|------------| +| enable |Indicates whether the MetaMux must be enabled. | Boolean |enable=1 |dGPU, Jetson| +|config-file|Pathname of the configuration file for gst-dsmetamux plugin | String |config-file: ./config_metamux.txt|dGPU, Jetson| + +The gst-dsmetamux configuration details are introduced in gst-dsmetamux plugin README. + +# Sample Models + +The sample application uses the following models as samples. + +|Model Name | Inference Plugin | source | +|-----------|-------------------|-----------------------------------------------------------| +|bodypose2d |nvinfer|https://github.com/NVIDIA-AI-IOT/deepstream_pose_estimation| +|YoloV4 |nvinfer|https://github.com/NVIDIA-AI-IOT/yolov4_deepstream| +|peoplenet|nvinferserver|https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet| +|Primary Car detection|nvinferserver|DeepStream SDK| +|Secondary Car maker|nvinferserver|DeepStream SDK| +|Secondary Car type|nvinferserver|DeepStream SDK| +|trafficcamnet|nvinferserver|https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/trafficcamnet| +|LPD|nvinferserver|https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/lpdnet| +|LPR|nvinferserver|https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/lprnet| + +# Sample Configuration Files + +The application will create new inferencing branch for the designated primary GIE. The secondary GIEs should identify the primary GIE on which they work by setting "operate-on-gie-id" in nvinfer or nvinfereserver configuration file. + +To make every inferencing branch unique and identifiable, the "unique-id" for every GIE should be different and unique. The gst-dsmetamux module will rely on the "unique-id" to identify the metadata comes from which model. + +There are five sample configurations in current project for reference. + +* The sample configuration for the open source YoloV4, bodypose2d and TAO car license plate identification models with nvinferserver. + + - Configuration folder + + tritonclient/sample/configs/apps/bodypose_yolo_lpr + + "source4_1080p_dec_parallel_infer.yml" is the application configuration file. The other configuration files are for different modules in the pipeline, the application configuration file uses these files to configure different modules. + + - Pipeline Graph: + + ![Yolov4_BODY_LPR](pipeline_0.png) + - App Command: + + `` + ./apps/deepstream-parallel-infer/deepstream-parallel-infer -c configs/apps/bodypose_yolo_lpr/source4_1080p_dec_parallel_infer.yml + `` + +* The sample configuration for the TAO vehicle classifications, carlicense plate identification and peopleNet models with nvinferserver. + + - Configuration folder + + tritonclient/sample/configs/apps/vehicle_lpr_analytic + + "source4_1080p_dec_parallel_infer.yml" is the application configuration file. The other configuration files are for different modules in the pipeline, the application configuration file uses these files to configure different modules. + + - Pipeline Graph: + + ![LPR_Vehicle_peopleNet](new_pipe.png) + + - App Command: + + `` + ./apps/deepstream-parallel-infer/deepstream-parallel-infer -c configs/apps/vehicle_lpr_analytic/source4_1080p_dec_parallel_infer.yml + `` +* The sample configuration for the TAO vehicle classifications, carlicense plate identification and peopleNet models with nvinferserver and nvinfer. + + - Configuration folder + + tritonclient/sample/configs/apps/vehicle0_lpr_analytic + + "source4_1080p_dec_parallel_infer.yml" is the application configuration file. The other configuration files are for different modules in the pipeline, the application configuration file uses these files to configure different modules. The vehicle branch uses nvinfer, the car plate and the peoplenet branches use nvinferserver. + + - Pipeline Graph: + + ![LPR_Vehicle_peopleNet](new_pipe.png) + + - App Command: + + `` + ./apps/deepstream-parallel-infer/deepstream-parallel-infer -c configs/apps/vehicle0_lpr_analytic/source4_1080p_dec_parallel_infer.yml + `` +* The sample configuration for the open source YoloV4, bodypose2d with nvinferserver and nvinfer. + + - Configuration folder + + tritonclient/sample/configs/apps/bodypose_yolo/ + + "source4_1080p_dec_parallel_infer.yml" is the application configuration file. The other configuration files are for different modules in the pipeline, the application configuration file uses these files to configure different modules. The bodypose branch uses nvinfer, the yolov4 branch use nvinferserver. The output streams is tiled. + + - Pipeline Graph: + + ![LPR_Vehicle_peopleNet](demo_pipe.png) + + - App Command: + + `` + ./apps/deepstream-parallel-infer/deepstream-parallel-infer -c configs/apps/bodypose_yolo/source4_1080p_dec_parallel_infer.yml + `` +* The sample configuration for the open source YoloV4, bodypose2d with nvinferserver and nvinfer. + + - Configuration folder + + tritonclient/sample/configs/apps/bodypose_yolo_win1/ + + "source4_1080p_dec_parallel_infer.yml" is the application configuration file. The other configuration files are for different modules in the pipeline, the application configuration file uses these files to configure different modules. The bodypose branch uses nvinfer, the yolov4 branch use nvinferserver. The output streams is source 2. + + - Pipeline Graph: + + ![LPR_Vehicle_peopleNet](demo_pipe_src2.png) + + - App Command: + + `` + ./apps/deepstream-parallel-infer/deepstream-parallel-infer -c configs/apps/bodypose_yolo_win1/source4_1080p_dec_parallel_infer.yml + `` diff --git a/deepstream_parallel_inference_app/common.png b/deepstream_parallel_inference_app/common.png new file mode 100755 index 0000000..08a4801 Binary files /dev/null and b/deepstream_parallel_inference_app/common.png differ diff --git a/deepstream_parallel_inference_app/demo_pipe.png b/deepstream_parallel_inference_app/demo_pipe.png new file mode 100755 index 0000000..a52d9e1 Binary files /dev/null and b/deepstream_parallel_inference_app/demo_pipe.png differ diff --git a/deepstream_parallel_inference_app/demo_pipe_src2.png b/deepstream_parallel_inference_app/demo_pipe_src2.png new file mode 100755 index 0000000..e4f3e87 Binary files /dev/null and b/deepstream_parallel_inference_app/demo_pipe_src2.png differ diff --git a/deepstream_parallel_inference_app/files.PNG b/deepstream_parallel_inference_app/files.PNG new file mode 100755 index 0000000..89285cf Binary files /dev/null and b/deepstream_parallel_inference_app/files.PNG differ diff --git a/deepstream_parallel_inference_app/new_pipe.png b/deepstream_parallel_inference_app/new_pipe.png new file mode 100644 index 0000000..970a343 Binary files /dev/null and b/deepstream_parallel_inference_app/new_pipe.png differ diff --git a/deepstream_parallel_inference_app/pipeline_0.png b/deepstream_parallel_inference_app/pipeline_0.png new file mode 100755 index 0000000..a9beac8 Binary files /dev/null and b/deepstream_parallel_inference_app/pipeline_0.png differ diff --git a/deepstream_parallel_inference_app/tritonclient/sample/Makefile b/deepstream_parallel_inference_app/tritonclient/sample/Makefile new file mode 100755 index 0000000..3489244 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/Makefile @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +all: + @make -C apps/deepstream-parallel-infer + @make -C gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo + @make -C gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser +clean: + @make clean -C apps/deepstream-parallel-infer + @make clean -C gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo + @make clean -C gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser diff --git a/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/CLA_LICENSE.md b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/CLA_LICENSE.md new file mode 100755 index 0000000..2581c66 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/CLA_LICENSE.md @@ -0,0 +1,25 @@ +# Individual Contributor License Agreement (CLA) + +Thank you for submitting your contributions to this project. + +By signing this CLA, you agree that the following terms apply to all of your past, present and future contributions to the project. + +## License. +You hereby represent that all present, past and future contributions are governed by the MIT License copyright statement. + +This entails that to the extent possible under law, you transfer all copyright and related or neighboring rights of the code or documents you contribute to the project itself or its maintainers. Furthermore you also represent that you have the authority to perform the above waiver with respect to the entirety of you contributions. + +## Moral Rights. +To the fullest extent permitted under applicable law, you hereby waive, and agree not to assert, all of your “moral rights” in or relating to your contributions for the benefit of the project. + +## Third Party Content. +If your Contribution includes or is based on any source code, object code, bug fixes, configuration changes, tools, specifications, documentation, data, materials, feedback, information or other works of authorship that were not authored by you (“Third Party Content”) or if you are aware of any third party intellectual property or proprietary rights associated with your Contribution (“Third Party Rights”), then you agree to include with the submission of your Contribution full details respecting such Third Party Content and Third Party Rights, including, without limitation, identification of which aspects of your Contribution contain Third Party Content or are associated with Third Party Rights, the owner/author of the Third Party Content and Third Party Rights, where you obtained the Third Party Content, and any applicable third party license terms or restrictions respecting the Third Party Content and Third Party Rights. For greater certainty, the foregoing obligations respecting the identification of Third Party Content and Third Party Rights do not apply to any portion of a Project that is incorporated into your Contribution to that same Project. + +## Representations. +You represent that, other than the Third Party Content and Third Party Rights identified by you in accordance with this Agreement, you are the sole author of your Contributions and are legally entitled to grant the foregoing licenses and waivers in respect of your Contributions. If your Contributions were created in the course of your employment with your past or present employer(s), you represent that such employer(s) has authorized you to make your Contributions on behalf of such employer(s) or such employer (s) has waived all of their right, title or interest in or to your Contributions. + +## Disclaimer. +To the fullest extent permitted under applicable law, your Contributions are provided on an "as is" basis, without any warranties or conditions, express or implied, including, without limitation, any implied warranties or conditions of non-infringement, merchantability or fitness for a particular purpose. You are not required to provide support for your Contributions, except to the extent you desire to provide support. + +## No Obligation. +You acknowledge that the maintainers of this project are under no obligation to use or incorporate your contributions into the project. The decision to use or incorporate your contributions into the project will be made at the sole discretion of the maintainers or their authorized delegates. \ No newline at end of file diff --git a/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/LICENSE.md b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/LICENSE.md new file mode 100755 index 0000000..59479b3 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/LICENSE.md @@ -0,0 +1,14 @@ +SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/Makefile b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/Makefile new file mode 100755 index 0000000..3346704 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/Makefile @@ -0,0 +1,72 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +APP:= deepstream-parallel-infer + +TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -) + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/bin/ + +ifeq ($(TARGET_DEVICE),aarch64) + CFLAGS:= -DPLATFORM_TEGRA +endif + +#SRCS:= $(wildcard *.c) $(wildcard *.cpp) +SRCS:= deepstream_parallel_infer_app.cpp \ + deepstream_parallel_infer_config_parser.cpp + +SRCS+= $(wildcard /opt/nvidia/deepstream/deepstream/sources/apps/apps-common/src/*.c) +SRCS+= $(wildcard /opt/nvidia/deepstream/deepstream/sources/apps/apps-common/src/deepstream-yaml/*.cpp) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 gstreamer-video-1.0 x11 json-glib-1.0 + +OBJS:= $(SRCS:.c=.o) +OBJS:= $(OBJS:.cpp=.o) + +CFLAGS+= -g -I./ -I/opt/nvidia/deepstream/deepstream/sources/apps/apps-common/includes \ + -I/opt/nvidia/deepstream/deepstream/sources/includes \ + -I /usr/local/cuda/include + +LIBS:= -L/usr/local/cuda/lib64/ -lcudart + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvdsgst_helper \ + -lnvdsgst_smartrecord -lnvds_utils -lnvds_msgbroker -lm -lyaml-cpp \ + -lcuda -lgstrtspserver-1.0 -ldl -Wl,-rpath,$(LIB_INSTALL_DIR) -lnvbufsurface + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) + +LIBS+= $(shell pkg-config --libs $(PKGS)) + +all: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +%.o: %.cpp $(INCS) Makefile + $(CXX) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CXX) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) diff --git a/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/README.md b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/README.md new file mode 100755 index 0000000..65472d8 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/README.md @@ -0,0 +1,24 @@ +# DeepStream Parallel Inference + +DeepStream Parallel Inference is one application which run inference in parallel. tee will send batched buffer to different models. nvdsmetamux will mux batch meta from different models which run in parallel. User can configure the pipeline in application configuration yaml file. User can select source ids which need inference in preprocess configuration file. User can select source ids which need mux in nvdsmetamux configuration file. + +## Prerequisites +DeepStream SDK 6.1.1 GA or above + + +## Getting Started: +To get started, please follow these steps. +1. Install [DeepStream](https://developer.nvidia.com/deepstream-sdk) on your platform, verify it is working by running deepstream-app. +2. Install nvdsmetamux. +3. Install triton client video template libraries. +4. Clone the repository preferably in `$DEEPSTREAM_DIR/sources/apps/sample_apps`. +5. Compile and run the program + + ``` + $ cd deepstream-parallel-infer/ + $ sudo make + $ sudo ./deepstream-parallel-infer -c +``` +Run parallel inference app with default configure file: $ ./deepstream-parallel-infer -c source4_1080p_dec_parallel_infer.yml + +For any issues or questions, please feel free to make a new post on the [DeepStreamSDK forums](https://forums.developer.nvidia.com/c/accelerated-computing/intelligent-video-analytics/deepstream-sdk/). diff --git a/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_metamux_yaml.cpp b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_metamux_yaml.cpp new file mode 100755 index 0000000..588a3c6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_metamux_yaml.cpp @@ -0,0 +1,62 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "deepstream_common.h" +#include "deepstream_config_yaml.h" +#include +#include +#include + +using std::cout; +using std::endl; + +gboolean +parse_metamux_yaml (NvDsMetaMuxConfig *config, gchar* cfg_file_path) +{ + gboolean ret = FALSE; + + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + for(YAML::const_iterator itr = configyml["meta-mux"].begin(); + itr != configyml["meta-mux"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + config->enable = itr->second.as(); + } else if (paramKey == "config-file") { + std::string temp = itr->second.as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, temp.c_str(), 1024); + config->config_file_path = (char*) malloc(sizeof(char) * 1024); + if (!get_absolute_file_path_yaml (cfg_file_path, str, + config->config_file_path)) { + g_printerr ("Error: Could not parse config-file-path in metamux.\n"); + g_free (str); + goto done; + } + g_free (str); + } else { + cout << "[WARNING] Unknown param found in metamux: " << paramKey << endl; + } + } + + ret = TRUE; +done: + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} diff --git a/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer.h b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer.h new file mode 100755 index 0000000..4288687 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer.h @@ -0,0 +1,232 @@ +/* +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __NVGSTDS_APP_H__ +#define __NVGSTDS_APP_H__ + +#include +#ifdef __cplusplus +extern "C" +{ +#endif + +#include + +#include "deepstream_app_version.h" +#include "deepstream_common.h" +#include "deepstream_config.h" +#include "deepstream_osd.h" +#include "deepstream_perf.h" +#include "deepstream_preprocess.h" +#include "deepstream_primary_gie.h" +#include "deepstream_sinks.h" +#include "deepstream_sources.h" +#include "deepstream_streammux.h" +#include "deepstream_tiled_display.h" +#include "deepstream_dsanalytics.h" +#include "deepstream_dsexample.h" +#include "deepstream_tracker.h" +#include "deepstream_secondary_gie.h" +#include "deepstream_c2d_msg.h" +#include "deepstream_image_save.h" + +#define MAX_PRIMARY_GIE_BINS (16) +#define MAX_PRE_PROCESS_BINS (16) +#define MAX_VIDEO_TEMPLATE_PROPS (32) + +typedef struct _AppCtx AppCtx; + +typedef void (*bbox_generated_callback) (AppCtx *appCtx, GstBuffer *buf, + NvDsBatchMeta *batch_meta, guint index); +typedef gboolean (*overlay_graphics_callback) (AppCtx *appCtx, GstBuffer *buf, + NvDsBatchMeta *batch_meta, guint index); + +typedef struct +{ + gboolean enable; + + gchar *config_file_path; +} NvDsMetaMuxConfig; + +typedef struct +{ + gboolean enable; + + guint num_customlib_props; + gchar *customlib_name; + gchar *customlib_props[MAX_VIDEO_TEMPLATE_PROPS]; +} NvDsVideoTemplateConfig; + +typedef struct +{ + GstElement *bin; + GstElement *tee; + GstElement *muxer; + gulong muxer_buffer_probe_id; + GstElement *source_tee[MAX_PRIMARY_GIE_BINS]; + GstElement *demuxer; + GstElement *streammux[MAX_PRIMARY_GIE_BINS]; + NvDsPrimaryGieBin primary_gie_bin[MAX_PRIMARY_GIE_BINS]; + NvDsTrackerBin tracker_bin[MAX_PRIMARY_GIE_BINS]; + NvDsSecondaryGieBin secondary_gie_bin[MAX_PRIMARY_GIE_BINS]; + NvDsDsAnalyticsBin dsanalytics_bin[MAX_PRIMARY_GIE_BINS]; + NvDsPreProcessBin preprocess_bin[MAX_PRIMARY_GIE_BINS]; +} NvDsParallelGieBin; + +typedef struct +{ + guint index; + gulong all_bbox_buffer_probe_id; + gulong primary_bbox_buffer_probe_id; + gulong fps_buffer_probe_id; + GstElement *bin; + GstElement *tee; + GstElement *sink_tee; + GstElement *msg_conv; + NvDsPreProcessBin preprocess_bin; + NvDsPrimaryGieBin primary_gie_bin; + NvDsOSDBin osd_bin; + NvDsSecondaryGieBin secondary_gie_bin; + NvDsTrackerBin tracker_bin; + NvDsSinkBin sink_bin; + NvDsSinkBin demux_sink_bin; + NvDsDsAnalyticsBin dsanalytics_bin; + NvDsDsExampleBin dsexample_bin; + AppCtx *appCtx; +} NvDsInstanceBin; + +typedef struct +{ + gulong primary_bbox_buffer_probe_id; + guint bus_id; + GstElement *pipeline; + NvDsSrcParentBin multi_src_bin; + NvDsParallelGieBin parallel_infer_bin; + NvDsInstanceBin instance_bins[MAX_SOURCE_BINS]; + NvDsInstanceBin demux_instance_bins[MAX_SOURCE_BINS]; + NvDsInstanceBin common_elements; + GstElement *tiler_tee; + NvDsTiledDisplayBin tiled_display_bin; + GstElement *demuxer; + NvDsDsExampleBin dsexample_bin; + AppCtx *appCtx; +} NvDsPipeline; + +typedef struct +{ + gint pgie_id; + gchar *src_ids; +} NvDsStrcIDConfig; + +typedef struct +{ + gboolean enable_perf_measurement; + gint file_loop; + gint pipeline_recreate_sec; + gboolean source_list_enabled; + guint total_num_sources; + guint num_source_sub_bins; + guint num_secondary_gie_num[MAX_PRIMARY_GIE_BINS]; + guint num_secondary_gie_sub_bins; + guint num_pre_process_sub_bins; + guint num_primary_gie_sub_bins; + guint num_src_ids_sub_bins; + guint num_tracker_sub_bins; + guint num_analysis_sub_bins; + guint num_sink_sub_bins; + guint num_message_consumers; + guint perf_measurement_interval_sec; + guint sgie_batch_size; + gchar *bbox_dir_path; + gchar *kitti_track_dir_path; + guint show_source; + + gchar **uri_list; + NvDsSourceConfig multi_source_config[MAX_SOURCE_BINS]; + NvDsStreammuxConfig streammux_config; + NvDsOSDConfig osd_config; + NvDsPreProcessConfig pre_process_sub_bin_config[MAX_PRIMARY_GIE_BINS]; + NvDsGieConfig primary_gie_sub_bin_config[MAX_PRIMARY_GIE_BINS]; + NvDsVideoTemplateConfig video_template_sub_bin_config[MAX_PRIMARY_GIE_BINS]; + NvDsMetaMuxConfig meta_mux_config; + NvDsTrackerConfig tracker_config[MAX_PRIMARY_GIE_BINS]; + NvDsStrcIDConfig srcids_config[MAX_PRIMARY_GIE_BINS]; + NvDsGieConfig secondary_gie_sub_bin_config[MAX_PRIMARY_GIE_BINS][MAX_SECONDARY_GIE_BINS]; + NvDsSinkSubBinConfig sink_bin_sub_bin_config[MAX_SINK_BINS]; + NvDsMsgConsumerConfig message_consumer_config[MAX_MESSAGE_CONSUMERS]; + NvDsTiledDisplayConfig tiled_display_config; + NvDsDsAnalyticsConfig dsanalytics_config[MAX_PRIMARY_GIE_BINS]; + NvDsDsExampleConfig dsexample_config; + NvDsSinkMsgConvBrokerConfig msg_conv_config; + NvDsImageSave image_save_config; + +} NvDsConfig; + +typedef struct +{ + gulong frame_num; +} NvDsInstanceData; + +struct _AppCtx +{ + gboolean version; + gboolean cintr; + gboolean show_bbox_text; + gboolean seeking; + gboolean quit; + gint person_class_id; + gint car_class_id; + gint return_value; + guint index; + gint active_source_index; + + GMutex app_lock; + GCond app_cond; + + NvDsPipeline pipeline; + NvDsConfig config; + NvDsConfig override_config; + NvDsInstanceData instance_data[MAX_SOURCE_BINS]; + NvDsC2DContext *c2d_ctx[MAX_MESSAGE_CONSUMERS]; + NvDsAppPerfStructInt perf_struct; + bbox_generated_callback bbox_generated_post_analytics_cb; + bbox_generated_callback all_bbox_generated_cb; + overlay_graphics_callback overlay_graphics_cb; + NvDsFrameLatencyInfo *latency_info; + GMutex latency_lock; + GThread *ota_handler_thread; + guint ota_inotify_fd; + guint ota_watch_desc; +}; + +/** + * Function to read properties from YML configuration file. + * + * @param[in] config pointer to @ref NvDsConfig + * @param[in] cfg_file_path path of configuration file. + * + * @return true if parsed successfully. + */ +gboolean +parse_config_file_yaml (NvDsConfig * config, gchar * cfg_file_path); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer_app.cpp b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer_app.cpp new file mode 100755 index 0000000..2e938ba --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer_app.cpp @@ -0,0 +1,1617 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "deepstream_parallel_infer.h" +#include "post_process/body_pose/post_process.cpp" +#include "nvds_version.h" + +#include +#include +#include + +#include "gstnvdsmeta.h" +#include "nvdsgstutils.h" +#include "nvbufsurface.h" +#include "nvdsmeta_schema.h" +#include "deepstream_perf.h" + +#include +#include +#include +#include +#include + +#define EPS 1e-6 + +#define MAX_DISPLAY_LEN 64 + +#define MAX_TIME_STAMP_LEN 32 + +#define PGIE_CLASS_ID_VEHICLE 0 +#define PGIE_CLASS_ID_PERSON 2 + +/* The muxer output resolution must be set if the input streams will be of + * different resolution. The muxer will scale all the input frames to this + * resolution. */ +#define MUXER_OUTPUT_WIDTH 1920 +#define MUXER_OUTPUT_HEIGHT 1080 +#define MAX_STR_LEN 2048 + +AppCtx *appCtx; +static guint cintr = FALSE; +static GMainLoop *main_loop = NULL; +static gchar **cfg_files = NULL; +static gboolean print_version = FALSE; +static gboolean show_bbox_text = FALSE; +static gboolean print_dependencies_version = FALSE; +static gboolean quit = FALSE; +static gint return_value = 0; +static guint num_input_uris; +static gint frame_interval = 30; + +template +using Vec1D = std::vector; + +template +using Vec2D = std::vector>; + +template +using Vec3D = std::vector>; + +gint frame_number = 0; + +#define MAX_STREAMS 64 + +typedef struct +{ + /** identifies the stream ID */ + guint32 stream_index; + gdouble fps[MAX_STREAMS]; + gdouble fps_avg[MAX_STREAMS]; + guint32 num_instances; + guint header_print_cnt; + GMutex fps_lock; + gpointer context; + + /** Test specific info */ + guint32 set_batch_size; +}DemoPerfCtx; + +typedef struct { + GMutex *lock; + int num_sources; +}LatencyCtx; + +/** + * callback function to print the performance numbers of each stream. + */ +static void +perf_cb (gpointer context, NvDsAppPerfStruct * str) +{ + DemoPerfCtx *thCtx = (DemoPerfCtx *) context; + + g_mutex_lock(&thCtx->fps_lock); + /** str->num_instances is == num_sources */ + guint32 numf = str->num_instances; + guint32 i; + + for (i = 0; i < numf; i++) { + thCtx->fps[i] = str->fps[i]; + thCtx->fps_avg[i] = str->fps_avg[i]; + } + thCtx->context = thCtx; + g_print ("**PERF: "); + for (i = 0; i < numf; i++) { + g_print ("%.2f (%.2f)\t", thCtx->fps[i], thCtx->fps_avg[i]); + } + g_print ("\n"); + g_mutex_unlock(&thCtx->fps_lock); +} + +/** + * callback function to print the latency of each component in the pipeline. + */ + +static GstPadProbeReturn +latency_measurement_buf_prob(GstPad * pad, GstPadProbeInfo * info, gpointer u_data) +{ + LatencyCtx *ctx = (LatencyCtx *) u_data; + static int batch_num = 0; + guint i = 0, num_sources_in_batch = 0; + if(nvds_enable_latency_measurement) + { + GstBuffer *buf = (GstBuffer *) info->data; + NvDsFrameLatencyInfo *latency_info = NULL; + g_mutex_lock (ctx->lock); + latency_info = (NvDsFrameLatencyInfo *) + calloc(1, ctx->num_sources * sizeof(NvDsFrameLatencyInfo));; + g_print("\n************BATCH-NUM = %d soure %d**************\n",batch_num,ctx->num_sources); + num_sources_in_batch = nvds_measure_buffer_latency(buf, latency_info); + + for(i = 0; i < num_sources_in_batch; i++) + { + g_print("Source id = %d Frame_num = %d Frame latency = %lf (ms) \n", + latency_info[i].source_id, + latency_info[i].frame_num, + latency_info[i].latency); + } + g_mutex_unlock (ctx->lock); + batch_num++; + } + + return GST_PAD_PROBE_OK; +} + +GST_DEBUG_CATEGORY (NVDS_APP); + +GOptionEntry entries[] = { + {"version", 'v', 0, G_OPTION_ARG_NONE, &print_version, + "Print DeepStreamSDK version", NULL} + , + {"tiledtext", 't', 0, G_OPTION_ARG_NONE, &show_bbox_text, + "Display Bounding box labels in tiled mode", NULL} + , + {"version-all", 0, 0, G_OPTION_ARG_NONE, &print_dependencies_version, + "Print DeepStreamSDK and dependencies version", NULL} + , + {"cfg-file", 'c', 0, G_OPTION_ARG_FILENAME_ARRAY, &cfg_files, + "Set the config file", NULL} + , + {NULL} + , +}; + +/*Method to parse information returned from the model*/ +std::tuple, Vec3D> +parse_objects_from_tensor_meta(NvDsInferTensorMeta *tensor_meta) +{ + Vec1D counts; + Vec3D peaks; + + float threshold = 0.1; + int window_size = 5; + int max_num_parts = 20; + int num_integral_samples = 7; + float link_threshold = 0.1; + int max_num_objects = 100; + + void *cmap_data = tensor_meta->out_buf_ptrs_host[0]; + NvDsInferDims &cmap_dims = tensor_meta->output_layers_info[0].inferDims; + void *paf_data = tensor_meta->out_buf_ptrs_host[1]; + NvDsInferDims &paf_dims = tensor_meta->output_layers_info[1].inferDims; + + /* Finding peaks within a given window */ + find_peaks(counts, peaks, cmap_data, cmap_dims, threshold, window_size, max_num_parts); + /* Non-Maximum Suppression */ + Vec3D refined_peaks = refine_peaks(counts, peaks, cmap_data, cmap_dims, window_size); + /* Create a Bipartite graph to assign detected body-parts to a unique person in the frame */ + Vec3D score_graph = paf_score_graph(paf_data, paf_dims, topology, counts, refined_peaks, num_integral_samples); + /* Assign weights to all edges in the bipartite graph generated */ + Vec3D connections = assignment(score_graph, topology, counts, link_threshold, max_num_parts); + /* Connecting all the Body Parts and Forming a Human Skeleton */ + Vec2D objects = connect_parts(connections, topology, counts, max_num_objects); + return {objects, refined_peaks}; +} + +/* MetaData to handle drawing onto the on-screen-display */ +static void +create_display_meta(Vec2D &objects, Vec3D &normalized_peaks, NvDsFrameMeta *frame_meta, int frame_width, int frame_height) +{ + int K = topology.size(); + int count = objects.size(); + NvDsBatchMeta *bmeta = frame_meta->base_meta.batch_meta; + NvDsDisplayMeta *dmeta = nvds_acquire_display_meta_from_pool(bmeta); + nvds_add_display_meta_to_frame(frame_meta, dmeta); + + for (auto &object : objects) + { + int C = object.size(); + for (int j = 0; j < C; j++) + { + int k = object[j]; + if (k >= 0) + { + auto &peak = normalized_peaks[j][k]; + int x = peak[1] * MUXER_OUTPUT_WIDTH; + int y = peak[0] * MUXER_OUTPUT_HEIGHT; + if (dmeta->num_circles == MAX_ELEMENTS_IN_DISPLAY_META) + { + dmeta = nvds_acquire_display_meta_from_pool(bmeta); + nvds_add_display_meta_to_frame(frame_meta, dmeta); + } + NvOSD_CircleParams &cparams = dmeta->circle_params[dmeta->num_circles]; + cparams.xc = x; + cparams.yc = y; + cparams.radius = 8; + cparams.circle_color = NvOSD_ColorParams{244, 67, 54, 1}; + cparams.has_bg_color = 1; + cparams.bg_color = NvOSD_ColorParams{0, 255, 0, 1}; + dmeta->num_circles++; + } + } + + for (int k = 0; k < K; k++) + { + int c_a = topology[k][2]; + int c_b = topology[k][3]; + if (object[c_a] >= 0 && object[c_b] >= 0) + { + auto &peak0 = normalized_peaks[c_a][object[c_a]]; + auto &peak1 = normalized_peaks[c_b][object[c_b]]; + int x0 = peak0[1] * MUXER_OUTPUT_WIDTH; + int y0 = peak0[0] * MUXER_OUTPUT_HEIGHT; + int x1 = peak1[1] * MUXER_OUTPUT_WIDTH; + int y1 = peak1[0] * MUXER_OUTPUT_HEIGHT; + if (dmeta->num_lines == MAX_ELEMENTS_IN_DISPLAY_META) + { + dmeta = nvds_acquire_display_meta_from_pool(bmeta); + nvds_add_display_meta_to_frame(frame_meta, dmeta); + } + NvOSD_LineParams &lparams = dmeta->line_params[dmeta->num_lines]; + lparams.x1 = x0; + lparams.x2 = x1; + lparams.y1 = y0; + lparams.y2 = y1; + lparams.line_width = 3; + lparams.line_color = NvOSD_ColorParams{0, 255, 0, 1}; + dmeta->num_lines++; + } + } + } +} + +/* body_pose_gie_src_pad_buffer_probe will extract metadata received from pgie + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +body_pose_gie_src_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) +{ + gchar *msg = NULL; + GstBuffer *buf = (GstBuffer *)info->data; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsMetaList *l_user = NULL; + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + + if (frame_meta->batch_id == 0) + g_print("Processing frame number = %d\t\n", frame_meta->frame_num); + + for (l_user = frame_meta->frame_user_meta_list; l_user != NULL; + l_user = l_user->next) + { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type == NVDSINFER_TENSOR_OUTPUT_META) + { + NvDsInferTensorMeta *tensor_meta = + (NvDsInferTensorMeta *)user_meta->user_meta_data; + Vec2D objects; + Vec3D normalized_peaks; + tie(objects, normalized_peaks) = parse_objects_from_tensor_meta(tensor_meta); + create_display_meta(objects, normalized_peaks, frame_meta, frame_meta->source_frame_width, frame_meta->source_frame_height); + } + } + + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) + { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)l_obj->data; + for (l_user = obj_meta->obj_user_meta_list; l_user != NULL; + l_user = l_user->next) + { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type == NVDSINFER_TENSOR_OUTPUT_META) + { + NvDsInferTensorMeta *tensor_meta = + (NvDsInferTensorMeta *)user_meta->user_meta_data; + Vec2D objects; + Vec3D normalized_peaks; + tie(objects, normalized_peaks) = parse_objects_from_tensor_meta(tensor_meta); + create_display_meta(objects, normalized_peaks, frame_meta, frame_meta->source_frame_width, frame_meta->source_frame_height); + } + } + } + } + return GST_PAD_PROBE_OK; +} + +static GstPadProbeReturn +yolov4_gie_src_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) +{ + gchar *msg = NULL; + GstBuffer *buf = (GstBuffer *)info->data; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsMetaList *l_user = NULL; + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + + for (l_user = frame_meta->frame_user_meta_list; l_user != NULL; + l_user = l_user->next) + { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type == NVDSINFER_TENSOR_OUTPUT_META) + { + NvDsInferTensorMeta *tensor_meta = + (NvDsInferTensorMeta *)user_meta->user_meta_data; + } + } + + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) + { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)l_obj->data; + for (l_user = obj_meta->obj_user_meta_list; l_user != NULL; + l_user = l_user->next) + { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type == NVDSINFER_TENSOR_OUTPUT_META) + { + NvDsInferTensorMeta *tensor_meta = + (NvDsInferTensorMeta *)user_meta->user_meta_data; + } + } + } + } + return GST_PAD_PROBE_OK; +} + +static void +generate_ts_rfc3339 (char *buf, int buf_size) +{ + time_t tloc; + struct tm tm_log; + struct timespec ts; + char strmsec[6]; //.nnnZ\0 + + clock_gettime (CLOCK_REALTIME, &ts); + memcpy (&tloc, (void *) (&ts.tv_sec), sizeof (time_t)); + gmtime_r (&tloc, &tm_log); + strftime (buf, buf_size, "%Y-%m-%dT%H:%M:%S", &tm_log); + int ms = ts.tv_nsec / 1000000; + g_snprintf (strmsec, sizeof (strmsec), ".%.3dZ", ms); + strncat (buf, strmsec, buf_size); +} + +static gpointer +meta_copy_func (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + NvDsEventMsgMeta *dstMeta = NULL; + + dstMeta = (NvDsEventMsgMeta *) g_memdup (srcMeta, sizeof (NvDsEventMsgMeta)); + + if (srcMeta->ts) + dstMeta->ts = g_strdup (srcMeta->ts); + + if (srcMeta->sensorStr) + dstMeta->sensorStr = g_strdup (srcMeta->sensorStr); + + if (srcMeta->objSignature.size > 0) { + dstMeta->objSignature.signature = (gdouble *) g_memdup (srcMeta->objSignature.signature, + srcMeta->objSignature.size); + dstMeta->objSignature.size = srcMeta->objSignature.size; + } + + if (srcMeta->objectId) { + dstMeta->objectId = g_strdup (srcMeta->objectId); + } + + if (srcMeta->extMsgSize > 0) { + if (srcMeta->objType == NVDS_OBJECT_TYPE_VEHICLE) { + NvDsVehicleObject *srcObj = (NvDsVehicleObject *) srcMeta->extMsg; + NvDsVehicleObject *obj = + (NvDsVehicleObject *) g_malloc0 (sizeof (NvDsVehicleObject)); + if (srcObj->type) + obj->type = g_strdup (srcObj->type); + if (srcObj->make) + obj->make = g_strdup (srcObj->make); + if (srcObj->model) + obj->model = g_strdup (srcObj->model); + if (srcObj->color) + obj->color = g_strdup (srcObj->color); + if (srcObj->license) + obj->license = g_strdup (srcObj->license); + if (srcObj->region) + obj->region = g_strdup (srcObj->region); + + dstMeta->extMsg = obj; + dstMeta->extMsgSize = sizeof (NvDsVehicleObject); + } else if (srcMeta->objType == NVDS_OBJECT_TYPE_PERSON) { + NvDsPersonObject *srcObj = (NvDsPersonObject *) srcMeta->extMsg; + NvDsPersonObject *obj = + (NvDsPersonObject *) g_malloc0 (sizeof (NvDsPersonObject)); + + obj->age = srcObj->age; + + if (srcObj->gender) + obj->gender = g_strdup (srcObj->gender); + if (srcObj->cap) + obj->cap = g_strdup (srcObj->cap); + if (srcObj->hair) + obj->hair = g_strdup (srcObj->hair); + if (srcObj->apparel) + obj->apparel = g_strdup (srcObj->apparel); + dstMeta->extMsg = obj; + dstMeta->extMsgSize = sizeof (NvDsPersonObject); + } + } + + return dstMeta; +} + +static void +meta_free_func (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + + g_free (srcMeta->ts); + g_free (srcMeta->sensorStr); + + if (srcMeta->objSignature.size > 0) { + g_free (srcMeta->objSignature.signature); + srcMeta->objSignature.size = 0; + } + + if (srcMeta->objectId) { + g_free (srcMeta->objectId); + } + + if (srcMeta->extMsgSize > 0) { + if (srcMeta->objType == NVDS_OBJECT_TYPE_VEHICLE) { + NvDsVehicleObject *obj = (NvDsVehicleObject *) srcMeta->extMsg; + if (obj->type) + g_free (obj->type); + if (obj->color) + g_free (obj->color); + if (obj->make) + g_free (obj->make); + if (obj->model) + g_free (obj->model); + if (obj->license) + g_free (obj->license); + if (obj->region) + g_free (obj->region); + } else if (srcMeta->objType == NVDS_OBJECT_TYPE_PERSON) { + NvDsPersonObject *obj = (NvDsPersonObject *) srcMeta->extMsg; + + if (obj->gender) + g_free (obj->gender); + if (obj->cap) + g_free (obj->cap); + if (obj->hair) + g_free (obj->hair); + if (obj->apparel) + g_free (obj->apparel); + } + g_free (srcMeta->extMsg); + srcMeta->extMsgSize = 0; + } + g_free (user_meta->user_meta_data); + user_meta->user_meta_data = NULL; +} + +static void +generate_vehicle_meta (gpointer data) +{ + NvDsVehicleObject *obj = (NvDsVehicleObject *) data; + + obj->type = g_strdup ("sedan"); + obj->color = g_strdup ("blue"); + obj->make = g_strdup ("Bugatti"); + obj->model = g_strdup ("M"); + obj->license = g_strdup ("XX1234"); + obj->region = g_strdup ("CA"); +} + +static void +generate_person_meta (gpointer data) +{ + NvDsPersonObject *obj = (NvDsPersonObject *) data; + obj->age = 45; + obj->cap = g_strdup ("none"); + obj->hair = g_strdup ("black"); + obj->gender = g_strdup ("male"); + obj->apparel = g_strdup ("formal"); +} + +static void +generate_event_msg_meta (gpointer data, gint class_id, + NvDsObjectMeta * obj_params) +{ + NvDsEventMsgMeta *meta = (NvDsEventMsgMeta *) data; + meta->sensorId = 0; + meta->placeId = 0; + meta->moduleId = 0; + meta->sensorStr = g_strdup ("sensor-0"); + + meta->ts = (gchar *) g_malloc0 (MAX_TIME_STAMP_LEN + 1); + meta->objectId = (gchar *) g_malloc0 (MAX_LABEL_SIZE); + + strncpy (meta->objectId, obj_params->obj_label, MAX_LABEL_SIZE); + + generate_ts_rfc3339 (meta->ts, MAX_TIME_STAMP_LEN); + + /* + * This demonstrates how to attach custom objects. + * Any custom object as per requirement can be generated and attached + * like NvDsVehicleObject / NvDsPersonObject. Then that object should + * be handled in payload generator library (nvmsgconv.cpp) accordingly. + */ + if (class_id == PGIE_CLASS_ID_VEHICLE) { + meta->type = NVDS_EVENT_MOVING; + meta->objType = NVDS_OBJECT_TYPE_VEHICLE; + meta->objClassId = PGIE_CLASS_ID_VEHICLE; + + NvDsVehicleObject *obj = + (NvDsVehicleObject *) g_malloc0 (sizeof (NvDsVehicleObject)); + generate_vehicle_meta (obj); + + meta->extMsg = obj; + meta->extMsgSize = sizeof (NvDsVehicleObject); + } else if (class_id == PGIE_CLASS_ID_PERSON) { + meta->type = NVDS_EVENT_ENTRY; + meta->objType = NVDS_OBJECT_TYPE_PERSON; + meta->objClassId = PGIE_CLASS_ID_PERSON; + + NvDsPersonObject *obj = + (NvDsPersonObject *) g_malloc0 (sizeof (NvDsPersonObject)); + generate_person_meta (obj); + + meta->extMsg = obj; + meta->extMsgSize = sizeof (NvDsPersonObject); + } +} +/* osd_sink_pad_buffer_probe will extract metadata received from OSD + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +osd_sink_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) +{ + GstBuffer *buf = (GstBuffer *)info->data; + guint num_rects = 0; + NvDsObjectMeta *obj_meta = NULL; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsDisplayMeta *display_meta = NULL; + gboolean is_first_object = TRUE; + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + is_first_object = TRUE; + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + int offset = 0; + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) + { + obj_meta = (NvDsObjectMeta *)(l_obj->data); + /** Generate NvDsEventMsgMeta for every object */ + if (is_first_object && !(frame_number % frame_interval)) { + NvDsEventMsgMeta *msg_meta = + (NvDsEventMsgMeta *) g_malloc0 (sizeof (NvDsEventMsgMeta)); + msg_meta->bbox.top = obj_meta->rect_params.top; + msg_meta->bbox.left = obj_meta->rect_params.left; + msg_meta->bbox.width = obj_meta->rect_params.width; + msg_meta->bbox.height = obj_meta->rect_params.height; + msg_meta->frameId = frame_number; + msg_meta->trackingId = obj_meta->object_id; + msg_meta->confidence = obj_meta->confidence; + generate_event_msg_meta (msg_meta, obj_meta->class_id, obj_meta); + + NvDsUserMeta *user_event_meta = + nvds_acquire_user_meta_from_pool (batch_meta); + if (user_event_meta) { + user_event_meta->user_meta_data = (void *) msg_meta; + user_event_meta->base_meta.meta_type = NVDS_EVENT_MSG_META; + user_event_meta->base_meta.copy_func = + (NvDsMetaCopyFunc) meta_copy_func; + user_event_meta->base_meta.release_func = + (NvDsMetaReleaseFunc) meta_free_func; + nvds_add_user_meta_to_frame (frame_meta, user_event_meta); + } else { + g_print ("Error in attaching event meta to buffer\n"); + } + } + } + display_meta = nvds_acquire_display_meta_from_pool(batch_meta); + + /* Parameters to draw text onto the On-Screen-Display */ + NvOSD_TextParams *txt_params = &display_meta->text_params[0]; + display_meta->num_labels = 1; + txt_params->display_text = (char *)g_malloc0(MAX_DISPLAY_LEN); + offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "Frame Number = %d", frame_number); + offset = snprintf(txt_params->display_text + offset, MAX_DISPLAY_LEN, ""); + + txt_params->x_offset = 10; + txt_params->y_offset = 12; + + txt_params->font_params.font_name = "Mono"; + txt_params->font_params.font_size = 10; + txt_params->font_params.font_color.red = 1.0; + txt_params->font_params.font_color.green = 1.0; + txt_params->font_params.font_color.blue = 1.0; + txt_params->font_params.font_color.alpha = 1.0; + + txt_params->set_bg_clr = 1; + txt_params->text_bg_clr.red = 0.0; + txt_params->text_bg_clr.green = 0.0; + txt_params->text_bg_clr.blue = 0.0; + txt_params->text_bg_clr.alpha = 1.0; + + nvds_add_display_meta_to_frame(frame_meta, display_meta); + } + frame_number++; + return GST_PAD_PROBE_OK; +} + +static gboolean +bus_call(GstBus *bus, GstMessage *msg, gpointer data) +{ + GMainLoop *main_loop = (GMainLoop *)data; + switch (GST_MESSAGE_TYPE(msg)) + { + case GST_MESSAGE_EOS: + g_print("End of Stream\n"); + g_main_loop_quit(main_loop); + break; + + case GST_MESSAGE_ERROR: + { + gchar *debug; + GError *error; + gst_message_parse_error(msg, &error, &debug); + g_printerr("ERROR from element %s: %s\n", + GST_OBJECT_NAME(msg->src), error->message); + if (debug) + g_printerr("Error details: %s\n", debug); + g_free(debug); + g_error_free(error); + g_main_loop_quit(main_loop); + break; + } + + default: + break; + } + return TRUE; +} + +gboolean +link_element_to_metamux_sink_pad (GstElement *metamux, GstElement *elem, + gint index) +{ + gboolean ret = FALSE; + GstPad *mux_sink_pad = NULL; + GstPad *src_pad = NULL; + gchar pad_name[16]; + + if (index >= 0) { + g_snprintf (pad_name, 16, "sink_%u", index); + pad_name[15] = '\0'; + } else { + strcpy (pad_name, "sink_%u"); + } + + mux_sink_pad = gst_element_get_request_pad (metamux, pad_name); + if (!mux_sink_pad) { + NVGSTDS_ERR_MSG_V ("Failed to get sink pad from metamux"); + goto done; + } + + src_pad = gst_element_get_static_pad (elem, "src"); + if (!src_pad) { + NVGSTDS_ERR_MSG_V ("Failed to get src pad from '%s'", + GST_ELEMENT_NAME (elem)); + goto done; + } + + if (gst_pad_link (src_pad, mux_sink_pad) != GST_PAD_LINK_OK) { + NVGSTDS_ERR_MSG_V ("Failed to link '%s' and '%s'", GST_ELEMENT_NAME (metamux), + GST_ELEMENT_NAME (elem)); + goto done; + } + + ret = TRUE; + +done: + if (mux_sink_pad) { + gst_object_unref (mux_sink_pad); + } + if (src_pad) { + gst_object_unref (src_pad); + } + return ret; +} + +gboolean +unlink_element_from_metamux_sink_pad (GstElement *metamux, GstElement *elem) +{ + gboolean ret = FALSE; + GstPad *mux_sink_pad = NULL; + GstPad *src_pad = NULL; + + src_pad = gst_element_get_static_pad (elem, "src"); + if (!src_pad) { + NVGSTDS_ERR_MSG_V ("Failed to get src pad from '%s'", + GST_ELEMENT_NAME (elem)); + goto done; + } + + mux_sink_pad = gst_pad_get_peer (src_pad); + if (!mux_sink_pad) { + NVGSTDS_ERR_MSG_V ("Failed to get sink pad from metamux"); + goto done; + } + + if (!gst_pad_unlink (src_pad, mux_sink_pad)) { + NVGSTDS_ERR_MSG_V ("Failed to unlink '%s' and '%s'", GST_ELEMENT_NAME (metamux), + GST_ELEMENT_NAME (elem)); + goto done; + } + + gst_element_release_request_pad(metamux, mux_sink_pad); + + ret = TRUE; + +done: + if (mux_sink_pad) { + gst_object_unref (mux_sink_pad); + } + if (src_pad) { + gst_object_unref (src_pad); + } + return ret; +} + +gboolean +link_streamdemux_to_streammux (NvDsParallelGieBin *bin, GstElement *demux, GstElement *mux, + gint index) +{ + gboolean ret = FALSE; + GstPad *mux_sink_pad = NULL; + GstPad *source_tee_src_pad = NULL; + GstElement *queue = NULL; + gchar pad_name[16]; + + if (!bin->source_tee[index]) { + bin->source_tee[index] = gst_element_factory_make (NVDS_ELEM_TEE, NULL); + if (!bin->source_tee[index]) { + NVGSTDS_ERR_MSG_V ("Failed to create 'infer_bin_source_tee'"); + goto done; + } + gst_bin_add (GST_BIN (bin->bin), bin->source_tee[index]); + + link_element_to_demux_src_pad (demux, bin->source_tee[index], index); + } + + queue = gst_element_factory_make (NVDS_ELEM_QUEUE, NULL); + if (!queue) { + NVGSTDS_ERR_MSG_V ("Could not create 'queue'"); + goto done; + } + gst_bin_add (GST_BIN (bin->bin), queue); + link_element_to_streammux_sink_pad (mux, queue, index); + + link_element_to_tee_src_pad (bin->source_tee[index], queue); + + ret = TRUE; + +done: + if (mux_sink_pad) { + gst_object_unref (mux_sink_pad); + } + if (source_tee_src_pad) { + gst_object_unref (source_tee_src_pad); + } + return ret; +} + +gboolean +create_primary_gie_videotemplate_bin (NvDsVideoTemplateConfig *config, NvDsPrimaryGieBin *bin) +{ + gboolean ret = FALSE; + guint i; + + bin->bin = gst_bin_new ("primary_gie_bin"); + if (!bin->bin) { + NVGSTDS_ERR_MSG_V ("Failed to create 'primary_gie_bin'"); + goto done; + } + + bin->nvvidconv = + gst_element_factory_make (NVDS_ELEM_VIDEO_CONV, "primary_gie_conv"); + if (!bin->nvvidconv) { + NVGSTDS_ERR_MSG_V ("Failed to create 'primary_gie_conv'"); + goto done; + } + + bin->queue = gst_element_factory_make (NVDS_ELEM_QUEUE, "primary_gie_queue"); + if (!bin->queue) { + NVGSTDS_ERR_MSG_V ("Failed to create 'primary_gie_queue'"); + goto done; + } + + bin->primary_gie = + gst_element_factory_make ("nvdsvideotemplate", "primary_gie"); + if (!bin->primary_gie) { + NVGSTDS_ERR_MSG_V ("Failed to create 'primary_gie'"); + goto done; + } + + g_object_set (G_OBJECT (bin->primary_gie), + "customlib-name", config->customlib_name, NULL); + + for (i = 0; i < config->num_customlib_props; i ++) { + g_object_set (G_OBJECT (bin->primary_gie), + "customlib-props", config->customlib_props[i], NULL); + } + + /* + g_object_set (G_OBJECT (bin->nvvidconv), "gpu-id", config->gpu_id, NULL); + g_object_set (G_OBJECT (bin->nvvidconv), "nvbuf-memory-type", + config->nvbuf_memory_type, NULL); + */ + + gst_bin_add_many (GST_BIN (bin->bin), bin->queue, + bin->nvvidconv, bin->primary_gie, NULL); + + NVGSTDS_LINK_ELEMENT (bin->queue, bin->nvvidconv); + + NVGSTDS_LINK_ELEMENT (bin->nvvidconv, bin->primary_gie); + + NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->primary_gie, "src"); + + NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->queue, "sink"); + + ret = TRUE; +done: + if (!ret) { + NVGSTDS_ERR_MSG_V ("%s failed", __func__); + } + return ret; +} + +#if 0 +static gboolean +create_parallel_infer_bin (guint num_sub_bins, NvDsConfig *config, + NvDsParallelGieBin *bin, AppCtx *appCtx) +{ + gboolean ret = FALSE; + GstElement *sink_elem = NULL; + GstElement *src_elem = NULL; + GstElement *nvvidconv = NULL, *caps_filter = NULL; + GstCaps *caps = NULL; + GstCapsFeatures *feature = NULL; + gchar name[50]; + guint i = 0; + + bin->bin = gst_bin_new ("parallel_infer_bin"); + if (!bin->bin) { + NVGSTDS_ERR_MSG_V ("Failed to create 'parallel_infer_bin'"); + goto done; + } + + bin->tee = gst_element_factory_make (NVDS_ELEM_TEE, "infer_bin_tee"); + if (!bin->tee) { + NVGSTDS_ERR_MSG_V ("Failed to create 'infer_bin_tee'"); + goto done; + } + gst_bin_add (GST_BIN (bin->bin), bin->tee); + + bin->muxer = gst_element_factory_make ("nvdsmetamux", "infer_bin_muxer"); + if (!bin->muxer) { + NVGSTDS_ERR_MSG_V ("Failed to create 'infer_bin_muxer'"); + goto done; + } + g_object_set (G_OBJECT (bin->muxer), "config-file", + GET_FILE_PATH (config->meta_mux_config.config_file_path), NULL); + + NVGSTDS_ELEM_ADD_PROBE (bin->muxer_buffer_probe_id, bin->muxer, "src", + body_pose_gie_src_pad_buffer_probe, GST_PAD_PROBE_TYPE_BUFFER, + appCtx); + + gst_bin_add (GST_BIN (bin->bin), bin->muxer); + + for (i = 0; i < num_sub_bins; i++) { + if (config->primary_gie_sub_bin_config[i].enable + || config->video_template_sub_bin_config[i].enable) { + if (config->video_template_sub_bin_config[i].enable) { + if (!create_primary_gie_videotemplate_bin (&config->video_template_sub_bin_config[i], + &bin->primary_gie_bin[i])) { + goto done; + } + } else { + if (!create_primary_gie_bin (&config->primary_gie_sub_bin_config[i], + &bin->primary_gie_bin[i])) { + goto done; + } + } + g_snprintf (name, sizeof (name), "primary_gie_%d_bin", i); + gst_element_set_name (bin->primary_gie_bin[i].bin, name); + gst_bin_add (GST_BIN (bin->bin), bin->primary_gie_bin[i].bin); + + sink_elem = bin->primary_gie_bin[i].bin; + src_elem = bin->primary_gie_bin[i].bin; + } + + if (config->pre_process_sub_bin_config[i].enable) { + if (!create_preprocess_bin (&config->pre_process_sub_bin_config[i], + &bin->preprocess_bin[i])) { + g_print ("creating preprocess bin failed\n"); + goto done; + } + g_snprintf (name, sizeof (name), "preprocess_%d_bin", i); + gst_element_set_name (bin->preprocess_bin[i].bin, name); + gst_bin_add (GST_BIN (bin->bin), bin->preprocess_bin[i].bin); + + if (sink_elem) { + NVGSTDS_LINK_ELEMENT (bin->preprocess_bin[i].bin, sink_elem); + } + + sink_elem = bin->preprocess_bin[i].bin; + } + + /* Add video convert to avoid parallel infer operate on the same batch meta */ + nvvidconv = gst_element_factory_make ("nvvideoconvert", NULL); + caps_filter = gst_element_factory_make ("capsfilter", NULL); + caps = + gst_caps_new_simple ("video/x-raw", + "width", G_TYPE_INT, 1920, + "height", G_TYPE_INT, 1082, + NULL); + feature = gst_caps_features_new ("memory:NVMM", NULL); + gst_caps_set_features (caps, 0, feature); + g_object_set (G_OBJECT (caps_filter), "caps", caps, NULL); + gst_bin_add (GST_BIN (bin->bin), nvvidconv); + gst_bin_add (GST_BIN (bin->bin), caps_filter); + NVGSTDS_LINK_ELEMENT (nvvidconv, caps_filter); + NVGSTDS_LINK_ELEMENT (caps_filter, sink_elem); + sink_elem = nvvidconv; + + link_element_to_tee_src_pad (bin->tee, sink_elem); + link_element_to_metamux_sink_pad (bin->muxer, src_elem, i); + } + + NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->tee, "sink"); + + NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->muxer, "src"); + + ret = TRUE; +done: + if (!ret) { + NVGSTDS_ERR_MSG_V ("%s failed", __func__); + } + return ret; +} +#endif + +/* Separate a config file entry with delimiters + * into strings. */ +std::vector split_string (std::string input) { + std::vector positions; + for (unsigned int i = 0; i < input.size(); i++) { + if (input[i] == ';') + positions.push_back(i); + } + std::vector ret; + int prev = 0; + for (auto &j: positions) { + std::string temp = input.substr(prev, j - prev); + ret.push_back(temp); + prev = j + 1; + } + ret.push_back(input.substr(prev, input.size() - prev)); + return ret; +} + +/* select streamdemux output sources, then link sources and streammux */ +static gboolean link_streamdemux_to_streammux(NvDsConfig *config, NvDsParallelGieBin *bin, int i){ + gboolean ret = FALSE; + if (config->primary_gie_sub_bin_config[i].unique_id != config->srcids_config[i].pgie_id) { + NVGSTDS_ERR_MSG_V ("pgieid %d != branch pgieid %d\n", + config->primary_gie_sub_bin_config[i].unique_id, + config->srcids_config[i].pgie_id); + return ret; + } else { + std::string str = config->srcids_config[i].src_ids; + std::vector vec = split_string (str); + for(int j = 0; j < vec.size(); j++) { + int id = std::stoi(vec[j]); + g_print("link_streamdemux_to_streammux, srid:%d, mux:%d\n", id, i); + if (!link_streamdemux_to_streammux (bin, bin->demuxer, bin->streammux[i], id)) { + NVGSTDS_ERR_MSG_V ("source %d cannot be linked to mux's sink pad %p\n", + id, bin->streammux[i]); + return ret; + } + } + } + return true; +} + +static gboolean +create_parallel_infer_bin (guint num_sub_bins, NvDsConfig *config, + NvDsParallelGieBin *bin, AppCtx *appCtx) +{ + gboolean ret = FALSE; + GstElement *sink_elem = NULL; + GstElement *src_elem = NULL; + GstElement *queue = NULL; + GstElement *nvvidconv = NULL, *caps_filter = NULL; + GstCaps *caps = NULL; + GstCapsFeatures *feature = NULL; + gchar name[50]; + guint i, j; + std::string str; + std::vector vec; + guint src_id_num; + + bin->bin = gst_bin_new ("parallel_infer_bin"); + if (!bin->bin) { + NVGSTDS_ERR_MSG_V ("Failed to create 'parallel_infer_bin'"); + goto done; + } + + bin->tee = gst_element_factory_make (NVDS_ELEM_TEE, "infer_bin_tee"); + if (!bin->tee) { + NVGSTDS_ERR_MSG_V ("Failed to create 'infer_bin_tee'"); + goto done; + } + gst_bin_add (GST_BIN (bin->bin), bin->tee); + + bin->muxer = gst_element_factory_make ("nvdsmetamux", "infer_bin_muxer"); + if (!bin->muxer) { + NVGSTDS_ERR_MSG_V ("Failed to create 'infer_bin_muxer'"); + goto done; + } + g_object_set (G_OBJECT (bin->muxer), "config-file", + GET_FILE_PATH (config->meta_mux_config.config_file_path), NULL); + + NVGSTDS_ELEM_ADD_PROBE (bin->muxer_buffer_probe_id, bin->muxer, "src", + body_pose_gie_src_pad_buffer_probe, GST_PAD_PROBE_TYPE_BUFFER, + appCtx); + + gst_bin_add (GST_BIN (bin->bin), bin->muxer); + sink_elem = bin->muxer; + + queue = gst_element_factory_make (NVDS_ELEM_QUEUE, NULL); + if (!queue) { + NVGSTDS_ERR_MSG_V ("Could not create 'queue'"); + goto done; + } + gst_bin_add (GST_BIN (bin->bin), queue); + link_element_to_metamux_sink_pad (bin->muxer, queue, 0); + sink_elem = queue; + + link_element_to_tee_src_pad (bin->tee, sink_elem); + + bin->demuxer = + gst_element_factory_make (NVDS_ELEM_STREAM_DEMUX, NULL); + if (!bin->demuxer) { + NVGSTDS_ERR_MSG_V ("Failed to create element 'demuxer'"); + goto done; + } + g_object_set (G_OBJECT (bin->demuxer), "per-stream-eos", TRUE, NULL); + gst_bin_add (GST_BIN (bin->bin), bin->demuxer); + sink_elem = bin->demuxer; + + queue = gst_element_factory_make (NVDS_ELEM_QUEUE, NULL); + if (!queue) { + NVGSTDS_ERR_MSG_V ("Could not create 'queue'"); + goto done; + } + gst_bin_add (GST_BIN (bin->bin), queue); + NVGSTDS_LINK_ELEMENT (queue, sink_elem); + sink_elem = queue; + + link_element_to_tee_src_pad (bin->tee, sink_elem); + + + for (i = 0; i < num_sub_bins; i++) { + sink_elem = src_elem = NULL; + + if (config->primary_gie_sub_bin_config[i].enable + || config->video_template_sub_bin_config[i].enable) { + + if (config->num_secondary_gie_sub_bins > 0 && config->num_secondary_gie_num[i] > 0) { + if (!create_secondary_gie_bin (config->num_secondary_gie_num[i], + config->primary_gie_sub_bin_config[i].unique_id, + config->secondary_gie_sub_bin_config[i], + &bin->secondary_gie_bin[i])) { + g_print("create_secondary_gie_bin failed"); + goto done; + } + g_snprintf (name, sizeof (name), "sgie_%d_bin", i); + gst_element_set_name (bin->secondary_gie_bin[i].bin, name); + gst_bin_add (GST_BIN (bin->bin), bin->secondary_gie_bin[i].bin); + sink_elem = bin->secondary_gie_bin[i].bin; + src_elem = bin->secondary_gie_bin[i].bin; + } + } + //add analysis + if (config->tracker_config[i].enable && config->dsanalytics_config[i].enable) { + if (!create_dsanalytics_bin (&config->dsanalytics_config[i], + &bin->dsanalytics_bin[i])) { + g_print ("creating dsanalytics bin failed\n"); + goto done; + } + + g_snprintf (name, sizeof (name), "analytics_%d_bin", i); + gst_element_set_name (bin->dsanalytics_bin[i].bin, name); + gst_bin_add (GST_BIN (bin->bin), bin->dsanalytics_bin[i].bin); + if (sink_elem) { + NVGSTDS_LINK_ELEMENT (bin->dsanalytics_bin[i].bin, sink_elem); + } + sink_elem = bin->dsanalytics_bin[i].bin; + if (!src_elem) { + src_elem = bin->dsanalytics_bin[i].bin; + } + } + //add tracker + if (config->tracker_config[i].enable) { + if (!create_tracking_bin (&config->tracker_config[i], + &bin->tracker_bin[i])) { + g_print ("creating tracker bin failed\n"); + goto done; + } + + g_snprintf (name, sizeof (name), "tracking_%d_bin", i); + gst_element_set_name (bin->tracker_bin[i].bin, name); + gst_bin_add (GST_BIN (bin->bin), + bin->tracker_bin[i].bin); + + if (sink_elem) { + NVGSTDS_LINK_ELEMENT (bin->tracker_bin[i].bin, sink_elem); + } + sink_elem = bin->tracker_bin[i].bin; + if (!src_elem) { + src_elem = bin->tracker_bin[i].bin; + } + } + + if (config->primary_gie_sub_bin_config[i].enable + || config->video_template_sub_bin_config[i].enable) { + if (config->video_template_sub_bin_config[i].enable) { + if (!create_primary_gie_videotemplate_bin (&config->video_template_sub_bin_config[i], + &bin->primary_gie_bin[i])) { + goto done; + } + } else { + if (!create_primary_gie_bin (&config->primary_gie_sub_bin_config[i], + &bin->primary_gie_bin[i])) { + goto done; + } + } + g_snprintf (name, sizeof (name), "primary_gie_%d_bin", i); + gst_element_set_name (bin->primary_gie_bin[i].bin, name); + gst_bin_add (GST_BIN (bin->bin), bin->primary_gie_bin[i].bin); + + if (sink_elem) { + NVGSTDS_LINK_ELEMENT (bin->primary_gie_bin[i].bin, sink_elem); + } + + sink_elem = bin->primary_gie_bin[i].bin; + if (!src_elem) { + src_elem = bin->primary_gie_bin[i].bin; + } + } + + if (config->pre_process_sub_bin_config[i].enable) { + if (!create_preprocess_bin (&config->pre_process_sub_bin_config[i], + &bin->preprocess_bin[i])) { + g_print ("creating preprocess bin failed\n"); + goto done; + } + g_snprintf (name, sizeof (name), "preprocess_%d_bin", i); + gst_element_set_name (bin->preprocess_bin[i].bin, name); + gst_bin_add (GST_BIN (bin->bin), bin->preprocess_bin[i].bin); + + if (sink_elem) { + NVGSTDS_LINK_ELEMENT (bin->preprocess_bin[i].bin, sink_elem); + } + + sink_elem = bin->preprocess_bin[i].bin; + } + + /* streamdemux and streammux to select source to inference */ + bin->streammux[i] = + gst_element_factory_make (NVDS_ELEM_STREAM_MUX, NULL); + if (!bin->streammux[i]) { + NVGSTDS_ERR_MSG_V ("Failed to create element 'streammux'"); + goto done; + } + gst_bin_add (GST_BIN (bin->bin), bin->streammux[i]); + if (config->streammux_config.is_parsed){ + if(!set_streammux_properties (&config->streammux_config, + bin->streammux[i])){ + NVGSTDS_WARN_MSG_V("Failed to set streammux properties"); + } + } + + str = config->srcids_config[i].src_ids; + vec = split_string (str); + src_id_num = vec.size(); + g_print("i:%d, src_id_num:%d\n", i, src_id_num); + g_object_set (G_OBJECT (bin->streammux[i]), "batch-size", src_id_num, NULL); + + if(!link_streamdemux_to_streammux(config, bin, i)){ + goto done; + } + + NVGSTDS_LINK_ELEMENT (bin->streammux[i], sink_elem); + + link_element_to_metamux_sink_pad (bin->muxer, src_elem, i+1); + } + + NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->tee, "sink"); + + NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->muxer, "src"); + + ret = TRUE; +done: + if (!ret) { + NVGSTDS_ERR_MSG_V ("%s failed", __func__); + } + return ret; +} + +static gboolean +add_and_link_broker_sink (AppCtx * appCtx) +{ + NvDsConfig *config = &appCtx->config; + /** Only first instance_bin broker sink + * employed as there's only one analytics path for N sources + * NOTE: There shall be only one [sink] group + * with type=6 (NV_DS_SINK_MSG_CONV_BROKER) + * a) Multiple of them does not make sense as we have only + * one analytics pipe generating the data for broker sink + * b) If Multiple broker sinks are configured by the user + * in config file, only the first in the order of + * appearance will be considered + * and others shall be ignored + * c) Ideally it should be documented (or obvious) that: + * multiple [sink] groups with type=6 (NV_DS_SINK_MSG_CONV_BROKER) + * is invalid + */ + NvDsInstanceBin *instance_bin = &appCtx->pipeline.instance_bins[0]; + NvDsPipeline *pipeline = &appCtx->pipeline; + + for (guint i = 0; i < config->num_sink_sub_bins; i++) { + if(config->sink_bin_sub_bin_config[i].type == NV_DS_SINK_MSG_CONV_BROKER) + { + /** add the broker sink bin to pipeline */ + if(!gst_bin_add (GST_BIN (pipeline->pipeline), instance_bin->sink_bin.sub_bins[i].bin)) { + return FALSE; + } + g_print("add_and_link_broker_sink\n"); + // link the broker sink bin to the sink tee + if (!link_element_to_tee_src_pad (instance_bin->sink_tee, instance_bin->sink_bin.sub_bins[i].bin)) { + return FALSE; + } + } + } + return TRUE; +} + +int main(int argc, char *argv[]) +{ + GOptionContext *ctx = NULL; + GOptionGroup *group = NULL; + GstElement *last_elem = NULL; + NvDsInstanceBin *instance_bin; + NvDsPipeline *pipeline; + NvDsConfig *config; + GstBus *bus = NULL; + guint bus_watch_id = 0; + GstPad *osd_sink_pad = NULL; + GError *error = NULL; + guint i; + const gchar *new_mux_str = NULL; + gboolean use_new_mux = FALSE; + + ctx = g_option_context_new ("Nvidia DeepStream Parallel Demo"); + group = g_option_group_new ("abc", NULL, NULL, NULL, NULL); + g_option_group_add_entries (group, entries); + + g_option_context_set_main_group (ctx, group); + g_option_context_add_group (ctx, gst_init_get_option_group ()); + + GST_DEBUG_CATEGORY_INIT (NVDS_APP, "NVDS_APP", 0, NULL); + + if (!g_option_context_parse (ctx, &argc, &argv, &error)) { + NVGSTDS_ERR_MSG_V ("%s", error->message); + return -1; + } + + if (print_version) { + g_print ("deepstream-app version %d.%d.%d\n", + NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO); + nvds_version_print (); + return 0; + } + + if (print_dependencies_version) { + g_print ("deepstream-app version %d.%d.%d\n", + NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO); + nvds_version_print (); + nvds_dependencies_version_print (); + return 0; + } + + if (!cfg_files) { + NVGSTDS_ERR_MSG_V ("Specify config file with -c option"); + return_value = -1; + goto done; + } + + appCtx = (AppCtx *)g_malloc0 (sizeof (AppCtx)); + appCtx->person_class_id = -1; + appCtx->car_class_id = -1; + appCtx->index = i; + appCtx->active_source_index = -1; + if (show_bbox_text) { + appCtx->show_bbox_text = TRUE; + } + + if (!parse_config_file_yaml (&appCtx->config, cfg_files[0])) { + NVGSTDS_ERR_MSG_V ("Failed to parse config file '%s'", cfg_files[0]); + appCtx->return_value = -1; + goto done; + } + + /* Standard GStreamer initialization */ + gst_init(&argc, &argv); + main_loop = g_main_loop_new(NULL, FALSE); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + instance_bin = &appCtx->pipeline.instance_bins[0]; + pipeline = &appCtx->pipeline; + config = &appCtx->config; + + pipeline->pipeline = gst_pipeline_new("deepstream-tensorrt-openpose-pipeline"); + + /* + * Add muxer and < N > source components to the pipeline based + * on the settings in configuration file. + */ + if (!create_multi_source_bin (config->num_source_sub_bins, + config->multi_source_config, &pipeline->multi_src_bin)) { + g_print ("creating multi source bin failed\n"); + goto done; + } + gst_bin_add (GST_BIN (pipeline->pipeline), pipeline->multi_src_bin.bin); + + /* if using new sreammux, nvvideocovnert will scale input resolutions to the same resolution */ + new_mux_str = g_getenv ("USE_NEW_NVSTREAMMUX"); + use_new_mux = !g_strcmp0 (new_mux_str, "yes"); + if (use_new_mux) { + GstCaps* caps = NULL; + gchar * caps_string = NULL; + GstCaps* fiter_caps = NULL; + char strCaps[MAX_STR_LEN] = {0}; + for (int i = 0; i < config->num_source_sub_bins; i++){ + g_object_get (G_OBJECT (pipeline->multi_src_bin.sub_bins[i].cap_filter1), "caps", &caps, NULL); + caps_string = gst_caps_to_string (caps); + snprintf(strCaps, MAX_STR_LEN-1, "%s,width=%d, height=%d", caps_string, + config->streammux_config.pipeline_width, config->streammux_config.pipeline_height); + fiter_caps = gst_caps_from_string (strCaps); + g_object_set (G_OBJECT (pipeline->multi_src_bin.sub_bins[i].cap_filter1), "caps", fiter_caps, NULL); + printf("strCaps:%s\n", strCaps); + + gst_caps_unref (caps); + gst_caps_unref (fiter_caps); + g_free (caps_string); + } + } + + if (config->streammux_config.is_parsed){ + if(!set_streammux_properties (&config->streammux_config, + pipeline->multi_src_bin.streammux)){ + NVGSTDS_WARN_MSG_V("Failed to set streammux properties"); + } + } + + if (!create_parallel_infer_bin (config->num_primary_gie_sub_bins, + config, &pipeline->parallel_infer_bin, appCtx)) { + g_print ("creating parallel infer bin failed\n"); + goto done; + } + gst_bin_add (GST_BIN (pipeline->pipeline), pipeline->parallel_infer_bin.bin); + last_elem = pipeline->parallel_infer_bin.bin; + NVGSTDS_LINK_ELEMENT (pipeline->multi_src_bin.bin, last_elem); + + /* Add common message converter */ + if (config->msg_conv_config.enable) { + NvDsSinkMsgConvBrokerConfig *convConfig = &config->msg_conv_config; + instance_bin->msg_conv = gst_element_factory_make (NVDS_ELEM_MSG_CONV, "common_msg_conv"); + if (!instance_bin->msg_conv) { + NVGSTDS_ERR_MSG_V ("Failed to create element 'common_msg_conv'"); + goto done; + } + + g_object_set (G_OBJECT(instance_bin->msg_conv), + "config", convConfig->config_file_path, + "msg2p-lib", (convConfig->conv_msg2p_lib ? convConfig->conv_msg2p_lib : "null"), + "payload-type", convConfig->conv_payload_type, + "comp-id", convConfig->conv_comp_id, + "debug-payload-dir", convConfig->debug_payload_dir, + "multiple-payloads", convConfig->multiple_payloads, + "msg2p-newapi", convConfig->conv_msg2p_new_api, + "frame-interval", convConfig->conv_frame_interval, + NULL); + + gst_bin_add (GST_BIN (pipeline->pipeline), + instance_bin->msg_conv); + + NVGSTDS_LINK_ELEMENT (last_elem, instance_bin->msg_conv); + last_elem = instance_bin->msg_conv; + } + + + + if (config->tiled_display_config.enable) { + if (config->tiled_display_config.columns * + config->tiled_display_config.rows < config->num_source_sub_bins) { + if (config->tiled_display_config.columns == 0) { + config->tiled_display_config.columns = + (guint) (sqrt (config->num_source_sub_bins) + 0.5); + } + config->tiled_display_config.rows = + (guint) ceil (1.0 * config->num_source_sub_bins / + config->tiled_display_config.columns); + NVGSTDS_WARN_MSG_V + ("Num of Tiles less than number of sources, readjusting to " + "%u rows, %u columns", config->tiled_display_config.rows, + config->tiled_display_config.columns); + } + + if (!create_tiled_display_bin (&config->tiled_display_config, + &pipeline->tiled_display_bin)) { + g_print ("creating tiled display bin failed\n"); + goto done; + } + gst_bin_add (GST_BIN (pipeline->pipeline), pipeline->tiled_display_bin.bin); + + if(config->show_source != -1){ + //default -1 means composite and show all sources + g_object_set(G_OBJECT(pipeline->tiled_display_bin.tiler), "show-source", config->show_source, NULL); + g_print("show-source:%d\n", config->show_source); + } + + + NVGSTDS_LINK_ELEMENT (last_elem, pipeline->tiled_display_bin.bin); + last_elem = pipeline->tiled_display_bin.bin; + osd_sink_pad = gst_element_get_static_pad(pipeline->tiled_display_bin.tiler, "sink"); + NvDsAppPerfStructInt *str = (NvDsAppPerfStructInt *)g_malloc0(sizeof(NvDsAppPerfStructInt)); + DemoPerfCtx *perf_ctx = (DemoPerfCtx *)g_malloc0(sizeof(DemoPerfCtx)); + g_mutex_init(&perf_ctx->fps_lock); + str->context = perf_ctx; + enable_perf_measurement (str, osd_sink_pad, config->num_source_sub_bins, 1, 0, perf_cb); + gst_object_unref(osd_sink_pad); + } + + if (config->osd_config.enable) { + if (!create_osd_bin (&config->osd_config, &instance_bin->osd_bin)) { + g_print ("creating osd bin failed\n"); + goto done; + } + gst_bin_add (GST_BIN (pipeline->pipeline), instance_bin->osd_bin.bin); + NVGSTDS_LINK_ELEMENT (last_elem, instance_bin->osd_bin.bin); + last_elem = instance_bin->osd_bin.bin; + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the osd element, since by that time, the buffer would have + * had got all the metadata. */ + osd_sink_pad = gst_element_get_static_pad(instance_bin->osd_bin.nvosd, "sink"); + if (!osd_sink_pad) + g_print("Unable to get sink pad\n"); + else { + gst_pad_add_probe(osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + osd_sink_pad_buffer_probe, NULL, NULL); + LatencyCtx *ctx = (LatencyCtx *)g_malloc0(sizeof(LatencyCtx)); + ctx->lock = (GMutex *)g_malloc0(sizeof(GMutex)); + ctx->num_sources = config->num_source_sub_bins; + gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + latency_measurement_buf_prob, ctx, NULL); + } + } + + //create sink_tee + instance_bin->sink_tee = gst_element_factory_make (NVDS_ELEM_TEE, "sink_tee"); + if (!instance_bin->sink_tee) { + NVGSTDS_ERR_MSG_V ("Failed to create 'sink_tee'"); + goto done; + } + gst_bin_add (GST_BIN (pipeline->pipeline), instance_bin->sink_tee); + NVGSTDS_LINK_ELEMENT (last_elem, instance_bin->sink_tee); + last_elem = instance_bin->sink_tee; + + if (!create_sink_bin (config->num_sink_sub_bins, + config->sink_bin_sub_bin_config, &instance_bin->sink_bin, 0)) { + g_print ("creating sink bin failed\n"); + goto done; + } + //x264enc will output one buffer after input 66 buffers at default, enable zerolatency property. + for(int i = 0; i < config->num_sink_sub_bins; i++){ + if(config->sink_bin_sub_bin_config[i].encoder_config.enc_type == NV_DS_ENCODER_TYPE_SW) + g_object_set (G_OBJECT (instance_bin->sink_bin.sub_bins[i].encoder), "tune", 0x4, NULL); + } + gst_bin_add (GST_BIN (pipeline->pipeline), instance_bin->sink_bin.bin); + NVGSTDS_LINK_ELEMENT (last_elem, instance_bin->sink_bin.bin); + + //link broker to sink-tee + add_and_link_broker_sink(appCtx); + + /* we add a message handler */ + bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline->pipeline)); + bus_watch_id = gst_bus_add_watch(bus, bus_call, main_loop); + gst_object_unref(bus); + + /* Set the pipeline to "playing" state */ + gst_element_set_state(pipeline->pipeline, GST_STATE_PLAYING); + + GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline->pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline"); + + /* Wait till pipeline encounters an error or EOS */ + g_print("Running...\n"); + g_main_loop_run(main_loop); + +done: + + g_print ("Quitting\n"); + if (bus_watch_id) { + g_print("Returned, stopping playback\n"); + gst_element_set_state(pipeline->pipeline, GST_STATE_NULL); + g_print("Deleting pipeline\n"); + gst_object_unref(GST_OBJECT(pipeline->pipeline)); + g_source_remove(bus_watch_id); + } + + if (appCtx) { + if (appCtx->return_value == -1) + return_value = -1; + g_free (appCtx); + } + + if (main_loop) { + g_main_loop_unref (main_loop); + } + + if (ctx) { + g_option_context_free (ctx); + } + + if (return_value == 0) { + g_print ("App run successful\n"); + } else { + g_print ("App run failed\n"); + } + + gst_deinit (); + + return return_value; +} diff --git a/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer_config_parser.cpp b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer_config_parser.cpp new file mode 100755 index 0000000..d2c5c73 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer_config_parser.cpp @@ -0,0 +1,615 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "deepstream_parallel_infer.h" +#include "deepstream_config_yaml.h" +#include + +#include +#include + +using std::cout; +using std::endl; + +static gboolean +parse_tests_yaml (NvDsConfig *config, gchar *cfg_file_path) +{ + gboolean ret = FALSE; + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + for(YAML::const_iterator itr = configyml["tests"].begin(); + itr != configyml["tests"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "file-loop") { + config->file_loop = itr->second.as(); + } else { + cout << "Unknown key " << paramKey << " for group tests" << endl; + } + } + + ret = TRUE; + + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} + +static gboolean +parse_app_yaml (NvDsConfig *config, gchar *cfg_file_path) +{ + gboolean ret = FALSE; + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + for(YAML::const_iterator itr = configyml["application"].begin(); + itr != configyml["application"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable-perf-measurement") { + config->enable_perf_measurement = + itr->second.as(); + } else if (paramKey == "perf-measurement-interval-sec") { + config->perf_measurement_interval_sec = + itr->second.as(); + } else if (paramKey == "gie-kitti-output-dir") { + std::string temp = itr->second.as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, temp.c_str(), 1024); + config->bbox_dir_path = (char*) malloc(sizeof(char) * 1024); + get_absolute_file_path_yaml (cfg_file_path, str, config->bbox_dir_path); + g_free(str); + } else if (paramKey == "kitti-track-output-dir") { + std::string temp = itr->second.as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, temp.c_str(), 1024); + config->kitti_track_dir_path = (char*) malloc(sizeof(char) * 1024); + get_absolute_file_path_yaml (cfg_file_path, str, config->kitti_track_dir_path); + g_free(str); + } + else { + cout << "Unknown key " << paramKey << " for group application" << endl; + } + } + + ret = TRUE; + + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} + +gboolean +parse_multi_preprocess_yaml (NvDsPreProcessConfig *config, std::string group, gchar *cfg_file_path) +{ + gboolean ret = FALSE; + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + for(YAML::const_iterator itr = configyml[group].begin(); + itr != configyml[group].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + config->enable = itr->second.as(); + } else if (paramKey == "config-file") { + std::string temp = itr->second.as(); + config->config_file_path = (char*) malloc(sizeof(char) * 1024); + std::strncpy (config->config_file_path, temp.c_str(), 1024); + } else { + cout << "[WARNING] Unknown param found in pre-process: " << paramKey << endl; + } + } + + ret = TRUE; + done: + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} + +gboolean +parse_video_template_yaml (NvDsVideoTemplateConfig *config, std::string group, gchar *cfg_file_path) +{ + gboolean ret = FALSE; + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + for(YAML::const_iterator itr = configyml[group].begin(); + itr != configyml[group].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + config->enable = itr->second.as(); + } else if (paramKey == "customlib-name") { + std::string temp = itr->second.as(); + config->customlib_name = (char*) malloc(sizeof(char) * 1024); + std::strncpy (config->customlib_name, temp.c_str(), 1024); + } else if (paramKey == "customlib-props") { + std::string temp = itr->second.as(); + config->customlib_props[config->num_customlib_props] = (char*) malloc(sizeof(char) * 1024); + std::strncpy (config->customlib_props[config->num_customlib_props], temp.c_str(), 1024); + config->num_customlib_props ++; + if (config->num_customlib_props == MAX_VIDEO_TEMPLATE_PROPS) { + NVGSTDS_ERR_MSG_V ("App supports max %d secondary GIEs", MAX_VIDEO_TEMPLATE_PROPS); + ret = FALSE; + goto done; + } + } else { + cout << "[WARNING] Unknown param found in pre-process: " << paramKey << endl; + } + } + + ret = TRUE; + done: + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} + +gboolean +parse_metamux_yaml (NvDsMetaMuxConfig *config, gchar* cfg_file_path) +{ + gboolean ret = FALSE; + + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + for(YAML::const_iterator itr = configyml["meta-mux"].begin(); + itr != configyml["meta-mux"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + config->enable = itr->second.as(); + } else if (paramKey == "config-file") { + std::string temp = itr->second.as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, temp.c_str(), 1024); + config->config_file_path = (char*) malloc(sizeof(char) * 1024); + if (!get_absolute_file_path_yaml (cfg_file_path, str, + config->config_file_path)) { + g_printerr ("Error: Could not parse config-file-path in metamux.\n"); + g_free (str); + goto done; + } + g_free (str); + } else { + cout << "[WARNING] Unknown param found in metamux: " << paramKey << endl; + } + } + + ret = TRUE; +done: + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} + +static gboolean +parse_sgie_yaml (NvDsConfig *config, NvDsGieConfig *gieConfig, std::string group, gchar *cfg_file_path, + gboolean enable) +{ + gboolean ret = FALSE; + gboolean parse_err = false; + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + int sgie_num = 0; + gboolean have_one_enable = false; + + for(YAML::const_iterator itr = configyml.begin(); + itr != configyml.end(); ++itr) + { + std::string paramKey = itr->first.as(); + parse_err = + !parse_gie_yaml (gieConfig + sgie_num, paramKey, cfg_file_path); + if(enable && gieConfig[sgie_num].enable){ + have_one_enable = true; + sgie_num++; + } + } + config->num_secondary_gie_num[config->num_secondary_gie_sub_bins] = sgie_num; + config->num_secondary_gie_sub_bins++; + + ret = TRUE; + + + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} + +static std::vector +split_csv_entries (std::string input) { + std::vector positions; + for (unsigned int i = 0; i < input.size(); i++) { + if (input[i] == ',') + positions.push_back(i); + } + std::vector ret; + int prev = 0; + for (auto &j: positions) { + std::string temp = input.substr(prev, j - prev); + ret.push_back(temp); + prev = j + 1; + } + ret.push_back(input.substr(prev, input.size() - prev)); + return ret; +} + +gboolean +parse_tiled_display_yaml (NvDsConfig *appConfig, NvDsTiledDisplayConfig *config, gchar *cfg_file_path) +{ + gboolean ret = FALSE; + + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + for(YAML::const_iterator itr = configyml["tiled-display"].begin(); + itr != configyml["tiled-display"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + config->enable = + (NvDsTiledDisplayEnable) itr->second.as(); + } else if (paramKey == "rows") { + config->rows = itr->second.as(); + } else if (paramKey == "columns") { + config->columns = itr->second.as(); + } else if (paramKey == "width") { + config->width = itr->second.as(); + } else if (paramKey == "height") { + config->height = itr->second.as(); + } else if (paramKey == "gpu-id") { + config->gpu_id = itr->second.as(); + } else if (paramKey == "nvbuf-memory-type") { + config->nvbuf_memory_type = itr->second.as(); + } else if (paramKey == "show-source") { + appConfig->show_source = itr->second.as(); + } else { + cout << "[WARNING] Unknown param found in tiled-display: " << paramKey << endl; + } + } + ret = TRUE; + + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} + +gboolean +parse_config_file_yaml (NvDsConfig *config, gchar *cfg_file_path) +{ + gboolean parse_err = false; + gboolean ret = FALSE; + gboolean enable = false; + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + std::string source_str = "source"; + std::string sink_str = "sink"; + std::string pre_process_str = "pre-process"; + std::string pgie_str = "primary-gie"; + std::string branch_str = "branch"; + std::string tracker_str = "tracker"; + std::string sgie_str = "secondary-gie"; + std::string video_template_str = "video-template"; + std::string msgcons_str = "message-consumer"; + std::string analytics_str = "nvds-analytics"; + config->source_list_enabled = FALSE; + config->show_source = -1; //default: show all source + + for(YAML::const_iterator itr = configyml.begin(); + itr != configyml.end(); ++itr) { + std::string paramKey = itr->first.as(); + + if (paramKey == "application") { + parse_err = !parse_app_yaml (config, cfg_file_path); + } + else if (paramKey == "source") { + if(configyml["source"]["csv-file-path"]) { + std::string csv_file_path = configyml["source"]["csv-file-path"].as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, csv_file_path.c_str(), 1024); + char *abs_csv_path = (char*) malloc(sizeof(char) * 1024); + get_absolute_file_path_yaml (cfg_file_path, str, abs_csv_path); + g_free(str); + + std::ifstream inputFile (abs_csv_path); + if (!inputFile.is_open()) { + cout << "Couldn't open CSV file " << abs_csv_path << endl; + } + std::string line, temp; + /* Separating header field and inserting as strings into the vector. + */ + getline(inputFile, line); + std::vector headers = split_csv_entries(line); + /*Parsing each csv entry as an input source */ + while(getline(inputFile, line)) { + std::vector source_values = split_csv_entries(line); + if (config->num_source_sub_bins == MAX_SOURCE_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d sources", MAX_SOURCE_BINS); + ret = FALSE; + goto done; + } + guint source_id = 0; + source_id = config->num_source_sub_bins; + parse_err = !parse_source_yaml (&config->multi_source_config[source_id], headers, source_values, cfg_file_path); + if (config->multi_source_config[source_id].enable) + config->num_source_sub_bins++; + } + } else { + NVGSTDS_ERR_MSG_V ("CSV file not specified\n"); + ret = FALSE; + goto done; + } + } + else if (paramKey == "streammux") { + parse_err = !parse_streammux_yaml(&config->streammux_config, cfg_file_path); + } + else if (paramKey == "osd") { + parse_err = !parse_osd_yaml(&config->osd_config, cfg_file_path); + } + else if (paramKey.compare(0, pre_process_str.size(), pre_process_str) == 0) { + if (config->num_pre_process_sub_bins == MAX_PRE_PROCESS_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d pre-process", MAX_PRE_PROCESS_BINS); + ret = FALSE; + goto done; + } + parse_err = + !parse_multi_preprocess_yaml (&config->pre_process_sub_bin_config[config-> + num_pre_process_sub_bins], + paramKey, cfg_file_path); + if (config->pre_process_sub_bin_config[config->num_pre_process_sub_bins].enable){ + config->num_pre_process_sub_bins++; + } + } + else if (paramKey.compare(0, pgie_str.size(), pgie_str) == 0) { + if (config->num_primary_gie_sub_bins == MAX_PRIMARY_GIE_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d secondary GIEs", MAX_PRIMARY_GIE_BINS); + ret = FALSE; + goto done; + } + parse_err = + !parse_gie_yaml (&config->primary_gie_sub_bin_config[config-> + num_primary_gie_sub_bins], + paramKey, cfg_file_path); + if (config->primary_gie_sub_bin_config[config->num_primary_gie_sub_bins].enable){ + config->num_primary_gie_sub_bins++; + } + } + else if (paramKey.compare(0, video_template_str.size(), video_template_str) == 0) { + if (config->num_primary_gie_sub_bins == MAX_PRIMARY_GIE_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d secondary GIEs", MAX_PRIMARY_GIE_BINS); + ret = FALSE; + goto done; + } + parse_err = + !parse_video_template_yaml (&config->video_template_sub_bin_config[config-> + num_primary_gie_sub_bins], + paramKey, cfg_file_path); + if (config->video_template_sub_bin_config[config->num_primary_gie_sub_bins].enable){ + config->num_primary_gie_sub_bins++; + } + } + else if (paramKey == "meta-mux") { + parse_err = !parse_metamux_yaml (&config->meta_mux_config, cfg_file_path); + } + else if (paramKey.compare(0, branch_str.size(), branch_str) == 0) { + if(configyml[paramKey]["pgie-id"].as()){ + config->srcids_config[config->num_src_ids_sub_bins].pgie_id = + configyml[paramKey]["pgie-id"].as(); + if(configyml[paramKey]["src-ids"]) { + std::string src_ids = configyml[paramKey]["src-ids"].as(); + int index = config->num_src_ids_sub_bins; + config->srcids_config[index].src_ids = (char*) calloc(sizeof(char) * src_ids.size(), sizeof(char)); + std::strncpy (config->srcids_config[index].src_ids , src_ids.c_str(), src_ids.size()); + g_print("src_ids:%s\n", config->srcids_config[index].src_ids); + } + } + config->num_src_ids_sub_bins++; + } + else if (paramKey.compare(0, tracker_str.size(), tracker_str) == 0) { + if (config->num_tracker_sub_bins == MAX_PRIMARY_GIE_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d tracker", MAX_PRIMARY_GIE_BINS); + ret = FALSE; + goto done; + } + + enable = configyml[paramKey]["enable"].as(); + if(configyml[paramKey]["cfg-file-path"]) { + std::string csv_file_path = configyml[paramKey]["cfg-file-path"].as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, csv_file_path.c_str(), 1024); + + char *abs_csv_path = (char*) malloc(sizeof(char) * 1024); + get_absolute_file_path_yaml (cfg_file_path, str, abs_csv_path); + g_free(str); + + parse_err = + !parse_tracker_yaml (&config->tracker_config[config-> + num_tracker_sub_bins], abs_csv_path); + g_free(abs_csv_path); + config->tracker_config[config->num_tracker_sub_bins].enable = enable; + config->num_tracker_sub_bins++; + } + } + else if (paramKey.compare(0, sgie_str.size(), sgie_str) == 0) { + if (config->num_secondary_gie_sub_bins == MAX_SECONDARY_GIE_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d secondary GIEs", MAX_SECONDARY_GIE_BINS); + ret = FALSE; + goto done; + } + enable = configyml[paramKey]["enable"].as(); + if( configyml[paramKey]["cfg-file-path"]) { + std::string csv_file_path = configyml[paramKey]["cfg-file-path"].as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, csv_file_path.c_str(), 1024); + char *abs_csv_path = (char*) malloc(sizeof(char) * 1024); + get_absolute_file_path_yaml (cfg_file_path, str, abs_csv_path); + g_free(str); + NvDsGieConfig* pCfg = config->secondary_gie_sub_bin_config[config-> + num_secondary_gie_sub_bins]; + parse_err = + !parse_sgie_yaml(config, pCfg, + paramKey, abs_csv_path, enable); + } + } + else if (paramKey.compare(0, sink_str.size(), sink_str) == 0) { + if (config->num_sink_sub_bins == MAX_SINK_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d sinks", MAX_SINK_BINS); + ret = FALSE; + goto done; + } + parse_err = + !parse_sink_yaml (&config-> + sink_bin_sub_bin_config[config->num_sink_sub_bins], paramKey, cfg_file_path); + if (config-> + sink_bin_sub_bin_config[config->num_sink_sub_bins].enable) { + config->num_sink_sub_bins++; + } + } + else if (paramKey.compare(0, msgcons_str.size(), msgcons_str) == 0) { + if (config->num_message_consumers == MAX_MESSAGE_CONSUMERS) { + NVGSTDS_ERR_MSG_V ("App supports max %d consumers", MAX_MESSAGE_CONSUMERS); + ret = FALSE; + goto done; + } + parse_err = !parse_msgconsumer_yaml ( + &config->message_consumer_config[config->num_message_consumers], + paramKey, cfg_file_path); + + if (config->message_consumer_config[config->num_message_consumers].enable) { + config->num_message_consumers++; + } + } + else if (paramKey == "tiled-display") { + parse_err = !parse_tiled_display_yaml (config, &config->tiled_display_config, cfg_file_path); + } + else if (paramKey == "img-save") { + parse_err = !parse_image_save_yaml (&config->image_save_config , cfg_file_path); + } + + else if (paramKey == "ds-example") { + parse_err = !parse_dsexample_yaml (&config->dsexample_config, cfg_file_path); + } + else if (paramKey == "message-converter") { + parse_err = !parse_msgconv_yaml (&config->msg_conv_config, paramKey, cfg_file_path); + } + else if (paramKey.compare(0, analytics_str.size(), analytics_str) == 0) { + if (config->num_analysis_sub_bins == MAX_PRIMARY_GIE_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d analysis", MAX_PRIMARY_GIE_BINS); + ret = FALSE; + goto done; + } + enable = configyml[paramKey]["enable"].as(); + config->dsanalytics_config[config->num_analysis_sub_bins].enable = enable; + if(configyml[paramKey]["cfg-file-path"]) { + std::string temp = configyml[paramKey]["cfg-file-path"].as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, temp.c_str(), 1024); + config->dsanalytics_config[config->num_analysis_sub_bins].config_file_path = + (char*) malloc(sizeof(char) * 1024); + if (!get_absolute_file_path_yaml (cfg_file_path, str, + config->dsanalytics_config[config->num_analysis_sub_bins].config_file_path)) { + g_printerr ("Error: Could not parse config-file in dsanalytics.\n"); + g_free (str); + goto done; + } + g_free (str); + } else { + cout << "[WARNING] Unknown param found in nvds-analytics: " << paramKey << endl; + } + config->num_analysis_sub_bins++; + } + else if (paramKey == "tests") { + parse_err = !parse_tests_yaml (config, cfg_file_path); + } + + if (parse_err) { + cout << "failed parsing" << endl; + goto done; + } + } + /* Updating batch size when source list is enabled */ + /* if (config->source_list_enabled == TRUE) { + // For streammux and pgie, batch size is set to number of sources + config->streammux_config.batch_size = config->num_source_sub_bins; + config->primary_gie_config.batch_size = config->num_source_sub_bins; + if (config->sgie_batch_size != 0) { + for (i = 0; i < config->num_secondary_gie_sub_bins; i++) { + config->secondary_gie_sub_bin_config[i].batch_size = config->sgie_batch_size; + } + } + } */ + + unsigned int i, j, k; + for (i = 0; i < config->num_secondary_gie_sub_bins; i++) { + for(int j = 0; j < config->num_secondary_gie_num[i]; j++ ){ + if (config->secondary_gie_sub_bin_config[i][j].unique_id == + config->primary_gie_sub_bin_config[i].unique_id) { + NVGSTDS_ERR_MSG_V ("Non unique gie ids found"); + ret = FALSE; + goto done; + } + } + } + + for (k = 0; k < config->num_secondary_gie_sub_bins; k++) { + for (i = 0; i < config->num_secondary_gie_num[k]; i++) { + for (j = i + 1; j < config->num_secondary_gie_num[k]; j++) { + if (config->secondary_gie_sub_bin_config[k][i].unique_id == + config->secondary_gie_sub_bin_config[k][j].unique_id) { + NVGSTDS_ERR_MSG_V ("Non unique gie id %d found", + config->secondary_gie_sub_bin_config[k][i].unique_id); + ret = FALSE; + goto done; + } + } + } + } + + for (i = 0; i < config->num_source_sub_bins; i++) { + if (config->multi_source_config[i].type == NV_DS_SOURCE_URI_MULTIPLE) { + if (config->multi_source_config[i].num_sources < 1) { + config->multi_source_config[i].num_sources = 1; + } + for (j = 1; j < config->multi_source_config[i].num_sources; j++) { + if (config->num_source_sub_bins == MAX_SOURCE_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d sources", MAX_SOURCE_BINS); + ret = FALSE; + goto done; + } + memcpy (&config->multi_source_config[config->num_source_sub_bins], + &config->multi_source_config[i], + sizeof (config->multi_source_config[i])); + config->multi_source_config[config->num_source_sub_bins].type = + NV_DS_SOURCE_URI; + config->multi_source_config[config->num_source_sub_bins].uri = + g_strdup_printf (config->multi_source_config[config-> + num_source_sub_bins].uri, j); + config->num_source_sub_bins++; + } + config->multi_source_config[i].type = NV_DS_SOURCE_URI; + config->multi_source_config[i].uri = + g_strdup_printf (config->multi_source_config[i].uri, 0); + } + } + + ret = TRUE; +done: + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} diff --git a/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/cover_table.hpp b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/cover_table.hpp new file mode 100755 index 0000000..7c2ff6c --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/cover_table.hpp @@ -0,0 +1,84 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +class CoverTable +{ +public: + CoverTable(int nrows, int ncols) : nrows(nrows), ncols(ncols) + { + rows.resize(nrows); + cols.resize(ncols); + } + + inline void coverRow(int row) + { + rows[row] = 1; + } + + inline void coverCol(int col) + { + cols[col] = 1; + } + + inline void uncoverRow(int row) + { + rows[row] = 0; + } + + inline void uncoverCol(int col) + { + cols[col] = 0; + } + + inline bool isCovered(int row, int col) const + { + return rows[row] || cols[col]; + } + + inline bool isRowCovered(int row) const + { + return rows[row]; + } + + inline bool isColCovered(int col) const + { + return cols[col]; + } + + inline void clear() + { + for (int i = 0; i < nrows; i++) + { + uncoverRow(i); + } + for (int j = 0; j < ncols; j++) + { + uncoverCol(j); + } + } + + const int nrows; + const int ncols; + +private: + std::vector rows; + std::vector cols; +}; diff --git a/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/munkres_algorithm.cpp b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/munkres_algorithm.cpp new file mode 100755 index 0000000..2bb73dd --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/munkres_algorithm.cpp @@ -0,0 +1,260 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pair_graph.hpp" +#include "cover_table.hpp" + +#include +#include +#include +#include +#include + +template +using Vec1D = std::vector; +template +using Vec2D = std::vector>; +template +using Vec3D = std::vector>; + +// Helper method to subtract the minimum row from cost_graph +void subtract_minimum_row(Vec2D &cost_graph, int nrows, int ncols) +{ + for (int i = 0; i < nrows; i++) + { + // Iterate the find the minimum + float min = cost_graph[i][0]; + for (int j = 0; j < ncols; j++) + { + float val = cost_graph[i][j]; + if (val < min) + { + min = val; + } + } + + // Subtract the Minimum + for (int j = 0; j < ncols; j++) + { + cost_graph[i][j] -= min; + } + } +} + +// Helper method to subtract the minimum col from cost_graph +void subtract_minimum_column(Vec2D &cost_graph, int nrows, int ncols) +{ + for (int j = 0; j < ncols; j++) + { + // Iterate and find the minimum + float min = cost_graph[0][j]; + for (int i = 0; i < nrows; i++) + { + float val = cost_graph[i][j]; + if (val < min) + { + min = val; + } + } + + // Subtract the minimum + for (int i = 0; i < nrows; i++) + { + cost_graph[i][j] -= min; + } + } +} + +void munkresStep1(Vec2D &cost_graph, PairGraph &star_graph, int nrows, + int ncols) +{ + for (int i = 0; i < nrows; i++) + { + for (int j = 0; j < ncols; j++) + { + if (!star_graph.isRowSet(i) && !star_graph.isColSet(j) && (cost_graph[i][j] == 0)) + { + star_graph.set(i, j); + } + } + } +} + +// Exits if '1' is returned +bool munkresStep2(const PairGraph &star_graph, CoverTable &cover_table) +{ + int k = + star_graph.nrows < star_graph.ncols ? star_graph.nrows : star_graph.ncols; + int count = 0; + for (int j = 0; j < star_graph.ncols; j++) + { + if (star_graph.isColSet(j)) + { + cover_table.coverCol(j); + count++; + } + } + return count >= k; +} + +bool munkresStep3(Vec2D &cost_graph, const PairGraph &star_graph, + PairGraph &prime_graph, CoverTable &cover_table, std::pair &p, + int nrows, int ncols) +{ + for (int i = 0; i < nrows; i++) + { + for (int j = 0; j < ncols; j++) + { + if (cost_graph[i][j] == 0 && !cover_table.isCovered(i, j)) + { + prime_graph.set(i, j); + if (star_graph.isRowSet(i)) + { + cover_table.coverRow(i); + cover_table.uncoverCol(star_graph.colForRow(i)); + } + else + { + p.first = i; + p.second = j; + return 1; + } + } + } + } + return 0; +}; + +void munkresStep4(PairGraph &star_graph, PairGraph &prime_graph, + CoverTable &cover_table, std::pair &p) +{ + // This process should be repeated until no star is found in prime's column + while (star_graph.isColSet(p.second)) + { + // First find and reset any star found in the prime's columns + std::pair s = {star_graph.rowForCol(p.second), p.second}; + star_graph.reset(s.first, s.second); + + // Set this prime to a star + star_graph.set(p.first, p.second); + + // Repeat the same process for prime in cleared star's row + p = {s.first, prime_graph.colForRow(s.first)}; + } + star_graph.set(p.first, p.second); + cover_table.clear(); + prime_graph.clear(); +} + +void munkresStep5(Vec2D &cost_graph, const CoverTable &cover_table, + int nrows, int ncols) +{ + bool valid = false; + float min; + for (int i = 0; i < nrows; i++) + { + for (int j = 0; j < ncols; j++) + { + if (!cover_table.isCovered(i, j)) + { + if (!valid) + { + min = cost_graph[i][j]; + valid = true; + } + else if (cost_graph[i][j] < min) + { + min = cost_graph[i][j]; + } + } + } + } + + for (int i = 0; i < nrows; i++) + { + if (cover_table.isRowCovered(i)) + { + for (int j = 0; j < ncols; j++) + { + cost_graph[i][j] += min; + } + } + } + for (int j = 0; j < ncols; j++) + { + if (!cover_table.isColCovered(j)) + { + for (int i = 0; i < nrows; i++) + { + cost_graph[i][j] -= min; + } + } + } +} + +void munkres_algorithm(Vec2D &cost_graph, PairGraph &star_graph, int nrows, + int ncols) +{ + PairGraph prime_graph(nrows, ncols); + CoverTable cover_table(nrows, ncols); + prime_graph.clear(); + cover_table.clear(); + star_graph.clear(); + + int step = 0; + if (ncols >= nrows) + { + subtract_minimum_row(cost_graph, nrows, ncols); + } + if (ncols > nrows) + { + step = 1; + } + + std::pair p; + bool done = false; + while (!done) + { + switch (step) + { + case 0: + subtract_minimum_column(cost_graph, nrows, ncols); + case 1: + munkresStep1(cost_graph, star_graph, nrows, ncols); + case 2: + if (munkresStep2(star_graph, cover_table)) + { + done = true; + break; + } + case 3: + if (!munkresStep3(cost_graph, star_graph, prime_graph, cover_table, p, + nrows, ncols)) + { + step = 5; + break; + } + case 4: + munkresStep4(star_graph, prime_graph, cover_table, p); + step = 2; + break; + case 5: + munkresStep5(cost_graph, cover_table, nrows, ncols); + step = 3; + break; + } + } +} diff --git a/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/pair_graph.hpp b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/pair_graph.hpp new file mode 100755 index 0000000..3f3c8b3 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/pair_graph.hpp @@ -0,0 +1,128 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +class PairGraph +{ +public: + PairGraph(int nrows, int ncols) : nrows(nrows), ncols(ncols) + { + this->rows.resize(nrows); + this->cols.resize(ncols); + } + + /** + * Returns the column index of the pair matching this row + */ + inline int colForRow(int row) const + { + return this->rows[row]; + } + + /** + * Returns the row index of the pair matching this column + */ + inline int rowForCol(int col) const + { + return this->cols[col]; + } + + /** + * Creates a pair between row and col + */ + inline void set(int row, int col) + { + this->rows[row] = col; + this->cols[col] = row; + } + + inline bool isRowSet(int row) const + { + return rows[row] >= 0; + } + + inline bool isColSet(int col) const + { + return cols[col] >= 0; + } + + inline bool isPair(int row, int col) + { + return rows[row] == col; + } + + /** + * Clears pair between row and col + */ + inline void reset(int row, int col) + { + this->rows[row] = -1; + this->cols[col] = -1; + } + + /** + * Clears all pairs in graph + */ + void clear() + { + for (int i = 0; i < this->nrows; i++) + { + this->rows[i] = -1; + } + for (int j = 0; j < this->ncols; j++) + { + this->cols[j] = -1; + } + } + + int numPairs() + { + int count = 0; + for (int i = 0; i < nrows; i++) + { + if (rows[i] >= 0) + { + count++; + } + } + return count; + } + + std::vector> pairs() + { + std::vector> p(numPairs()); + int count = 0; + for (int i = 0; i < nrows; i++) + { + if (isRowSet(i)) + { + p[count++] = {i, colForRow(i)}; + } + } + return p; + } + + const int nrows; + const int ncols; + +private: + std::vector rows; + std::vector cols; +}; diff --git a/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/post_process.cpp b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/post_process.cpp new file mode 100755 index 0000000..11352de --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/post_process.cpp @@ -0,0 +1,428 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pair_graph.hpp" +#include "cover_table.hpp" +#include "munkres_algorithm.cpp" + +#include +#include +#include + +//#include "gstnvdsmeta.h" +//#include "gstnvdsinfer.h" +#include "nvdsgstutils.h" +#include "nvbufsurface.h" + +#include +#include +#include +#include +#include + +#define EPS 1e-6 + +template +using Vec1D = std::vector; +template +using Vec2D = std::vector>; +template +using Vec3D = std::vector>; + +static const int M = 2; + +static Vec2D topology{ + {0, 1, 15, 13}, + {2, 3, 13, 11}, + {4, 5, 16, 14}, + {6, 7, 14, 12}, + {8, 9, 11, 12}, + {10, 11, 5, 7}, + {12, 13, 6, 8}, + {14, 15, 7, 9}, + {16, 17, 8, 10}, + {18, 19, 1, 2}, + {20, 21, 0, 1}, + {22, 23, 0, 2}, + {24, 25, 1, 3}, + {26, 27, 2, 4}, + {28, 29, 3, 5}, + {30, 31, 4, 6}, + {32, 33, 17, 0}, + {34, 35, 17, 5}, + {36, 37, 17, 6}, + {38, 39, 17, 11}, + {40, 41, 17, 12}}; + +/* Method to find peaks in the output tensor. 'window_size' represents how many pixels we are considering at once to find a maximum value, or a ‘peak’. + Once we find a peak, we mark it using the ‘is_peak’ boolean in the inner loop and assign this maximum value to the center pixel of our window. + This is then repeated until we cover the entire frame. */ +void find_peaks(Vec1D &counts_out, Vec3D &peaks_out, void *cmap_data, + NvDsInferDims &cmap_dims, float threshold, int window_size, int max_count) +{ + int w = window_size / 2; + int width = cmap_dims.d[2]; + int height = cmap_dims.d[1]; + + counts_out.assign(cmap_dims.d[0], 0); + peaks_out.assign(cmap_dims.d[0], Vec2D(max_count, Vec1D(M, + 0))); + + for (unsigned int c = 0; c < cmap_dims.d[0]; c++) + { + int count = 0; + float *cmap_data_c = (float *)cmap_data + c * width * height; + + for (int i = 0; i < height && count < max_count; i++) + { + for (int j = 0; j < width && count < max_count; j++) + { + float value = cmap_data_c[i * width + j]; + + if (value < threshold) + continue; + + int ii_min = i - w; + int jj_min = j - w; + int ii_max = i + w + 1; + int jj_max = j + w + 1; + + if (ii_min < 0) + ii_min = 0; + if (ii_max > height) + ii_max = height; + if (jj_min < 0) + jj_min = 0; + if (jj_max > width) + jj_max = width; + + bool is_peak = true; + for (int ii = ii_min; ii < ii_max; ii++) + { + for (int jj = jj_min; jj < jj_max; jj++) + { + if (cmap_data_c[ii * width + jj] > value) + { + is_peak = false; + } + } + } + + if (is_peak) + { + peaks_out[c][count][0] = i; + peaks_out[c][count][1] = j; + count++; + } + } + } + + counts_out[c] = count; + } +} + +/* Normalize the peaks found in 'find_peaks' and apply non-maximal suppression*/ +Vec3D +refine_peaks(Vec1D &counts, + Vec3D &peaks, void *cmap_data, NvDsInferDims &cmap_dims, + int window_size) +{ + int w = window_size / 2; + int width = cmap_dims.d[2]; + int height = cmap_dims.d[1]; + + Vec3D refined_peaks(peaks.size(), Vec2D(peaks[0].size(), + Vec1D(peaks[0][0].size(), 0))); + + for (unsigned int c = 0; c < cmap_dims.d[0]; c++) + { + int count = counts[c]; + auto &refined_peaks_a_bc = refined_peaks[c]; + auto &peaks_a_bc = peaks[c]; + float *cmap_data_c = (float *)cmap_data + c * width * height; + + for (int p = 0; p < count; p++) + { + auto &refined_peak = refined_peaks_a_bc[p]; + auto &peak = peaks_a_bc[p]; + + int i = peak[0]; + int j = peak[1]; + float weight_sum = 0.0f; + + for (int ii = i - w; ii < i + w + 1; ii++) + { + int ii_idx = ii; + + if (ii < 0) + ii_idx = -ii; + else if (ii >= height) + ii_idx = height - (ii - height) - 2; + + for (int jj = j - w; jj < j + w + 1; jj++) + { + int jj_idx = jj; + + if (jj < 0) + jj_idx = -jj; + else if (jj >= width) + jj_idx = width - (jj - width) - 2; + + float weight = cmap_data_c[ii_idx * width + jj_idx]; + refined_peak[0] += weight * ii; + refined_peak[1] += weight * jj; + weight_sum += weight; + } + } + + refined_peak[0] /= weight_sum; + refined_peak[1] /= weight_sum; + refined_peak[0] += 0.5; + refined_peak[1] += 0.5; + refined_peak[0] /= height; + refined_peak[1] /= width; + } + } + + return refined_peaks; +} + +/* Create a bipartite graph to assign detected body-parts to a unique person in the frame. This method also takes care of finding the line integral to assign scores + to these points */ +Vec3D +paf_score_graph(void *paf_data, NvDsInferDims &paf_dims, + Vec2D &topology, Vec1D &counts, + Vec3D &peaks, int num_integral_samples) +{ + int K = topology.size(); + int H = paf_dims.d[1]; + int W = paf_dims.d[2]; + int max_count = peaks[0].size(); + Vec3D score_graph(K, Vec2D(max_count, Vec1D(max_count, 0))); + + for (int k = 0; k < K; k++) + { + auto &score_graph_nk = score_graph[k]; + auto &paf_i_idx = topology[k][0]; + auto &paf_j_idx = topology[k][1]; + auto &cmap_a_idx = topology[k][2]; + auto &cmap_b_idx = topology[k][3]; + float *paf_i = (float *)paf_data + paf_i_idx * H * W; + float *paf_j = (float *)paf_data + paf_j_idx * H * W; + + auto &counts_a = counts[cmap_a_idx]; + auto &counts_b = counts[cmap_b_idx]; + auto &peaks_a = peaks[cmap_a_idx]; + auto &peaks_b = peaks[cmap_b_idx]; + + for (int a = 0; a < counts_a; a++) + { + // Point A + float pa_i = peaks_a[a][0] * H; + float pa_j = peaks_a[a][1] * W; + + for (int b = 0; b < counts_b; b++) + { + // Point B + float pb_i = peaks_b[b][0] * H; + float pb_j = peaks_b[b][1] * W; + + // Vector from Point A to Point B + float pab_i = pb_i - pa_i; + float pab_j = pb_j - pa_j; + + // Normalized Vector from Point A to Point B + float pab_norm = sqrtf(pab_i * pab_i + pab_j * pab_j) + EPS; + float uab_i = pab_i / pab_norm; + float uab_j = pab_j / pab_norm; + + float integral = 0.0; + float increment = 1.0f / num_integral_samples; + + for (int t = 0; t < num_integral_samples; t++) + { + // Integral Point T + float progress = (float)t / (float)num_integral_samples; + float pt_i = pa_i + progress * pab_i; + float pt_j = pa_j + progress * pab_j; + + // Convert to Integer + int pt_i_int = (int)pt_i; + int pt_j_int = (int)pt_j; + + // Edge cases for if the point is out of bounds, just skip them + if (pt_i_int < 0) + continue; + if (pt_i_int > H) + continue; + if (pt_j_int < 0) + continue; + if (pt_j_int > W) + continue; + + // Vector at integral point + float pt_paf_i = paf_i[pt_i_int * W + pt_j_int]; + float pt_paf_j = paf_j[pt_i_int * W + pt_j_int]; + + // Dot Product Normalized A->B with PAF Vector + float dot = pt_paf_i * uab_i + pt_paf_j * uab_j; + integral += dot; + + progress += increment; + } + + // Normalize the integral with respect to the number of samples + integral /= num_integral_samples; + score_graph_nk[a][b] = integral; + } + } + } + return score_graph; +} + +/* + This method takes care of solving the graph assignment problem using Munkres algorithm. Munkres algorithm is defind in 'munkres_algorithm.cpp' + */ + +Vec3D +assignment(Vec3D &score_graph, + Vec2D &topology, Vec1D &counts, float score_threshold, int max_count) +{ + int K = topology.size(); + Vec3D connections(K, Vec2D(M, Vec1D(max_count, -1))); + + Vec3D cost_graph = score_graph; + for (Vec2D &cg_iter1 : cost_graph) + for (Vec1D &cg_iter2 : cg_iter1) + for (float &cg_iter3 : cg_iter2) + cg_iter3 = -cg_iter3; + auto &cost_graph_out_a = cost_graph; + + for (int k = 0; k < K; k++) + { + int cmap_a_idx = topology[k][2]; + int cmap_b_idx = topology[k][3]; + int nrows = counts[cmap_a_idx]; + int ncols = counts[cmap_b_idx]; + auto star_graph = PairGraph(nrows, ncols); + auto &cost_graph_out_a_nk = cost_graph_out_a[k]; + munkres_algorithm(cost_graph_out_a_nk, star_graph, nrows, ncols); + + auto &connections_a_nk = connections[k]; + auto &score_graph_a_nk = score_graph[k]; + + for (int i = 0; i < nrows; i++) + { + for (int j = 0; j < ncols; j++) + { + if (star_graph.isPair(i, j) && score_graph_a_nk[i][j] > score_threshold) + { + connections_a_nk[0][i] = j; + connections_a_nk[1][j] = i; + } + } + } + } + return connections; +} + +/* This method takes care of connecting all the body parts detected to each other + after finding the relationships between them in the 'assignment' method */ +Vec2D +connect_parts( + Vec3D &connections, Vec2D &topology, Vec1D &counts, + int max_count) +{ + int K = topology.size(); + int C = counts.size(); + + Vec2D visited(C, Vec1D(max_count, 0)); + + Vec2D objects(max_count, Vec1D(C, -1)); + + int num_objects = 0; + for (int c = 0; c < C; c++) + { + if (num_objects >= max_count) + { + break; + } + + int count = counts[c]; + + for (int i = 0; i < count; i++) + { + if (num_objects >= max_count) + { + break; + } + + std::queue> q; + bool new_object = false; + q.push({c, i}); + + while (!q.empty()) + { + auto node = q.front(); + q.pop(); + int c_n = node.first; + int i_n = node.second; + + if (visited[c_n][i_n]) + { + continue; + } + + visited[c_n][i_n] = 1; + new_object = true; + objects[num_objects][c_n] = i_n; + + for (int k = 0; k < K; k++) + { + int c_a = topology[k][2]; + int c_b = topology[k][3]; + + if (c_a == c_n) + { + int i_b = connections[k][0][i_n]; + if (i_b >= 0) + { + q.push({c_b, i_b}); + } + } + + if (c_b == c_n) + { + int i_a = connections[k][1][i_n]; + if (i_a >= 0) + { + q.push({c_a, i_a}); + } + } + } + } + + if (new_object) + { + num_objects++; + } + } + } + + objects.resize(num_objects); + return objects; +} diff --git a/deepstream_parallel_inference_app/tritonclient/sample/build.sh b/deepstream_parallel_inference_app/tritonclient/sample/build.sh new file mode 100755 index 0000000..cb50060 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/build.sh @@ -0,0 +1,14 @@ + +make + +## enable NVSTREAMMUX_ADAPTIVE_BATCHING +if [ x"$NVSTREAMMUX_ADAPTIVE_BATCHING" != x"yes" ]; then + export NVSTREAMMUX_ADAPTIVE_BATCHING=yes + rm -rf ~/.cache/gstreamer-1.0/ + echo "export NVSTREAMMUX_ADAPTIVE_BATCHING=yes" +fi + +## dict.txt is label file for LPR model +if [ ! -f dict.txt ]; then + wget 'https://api.ngc.nvidia.com/v2/models/nvidia/tao/lprnet/versions/deployable_v1.0/files/us_lp_characters.txt' -O dict.txt +fi diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/US_LPD/config_uslpd_inferserver.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/US_LPD/config_uslpd_inferserver.txt new file mode 100755 index 0000000..4050359 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/US_LPD/config_uslpd_inferserver.txt @@ -0,0 +1,76 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 4 + gpu_ids: [0] + max_batch_size: 16 + backend { + triton { + model_name: "US_LPD" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + channel_offsets: [0.0,0.0,0.0] + } + } + + postprocess { + labelfile_path: "../../../../tritonserver/models/US_LPD/usa_lpd_label.txt" + detection { + num_detected_classes: 1 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 6 + } +} +input_control { + operate_on_gie_id: 3 + process_mode: PROCESS_MODE_CLIP_OBJECTS + secondary_reinfer_interval: 0 + object_control { + bbox_filter { + min_width: 40 + min_height: 30 + } + } +} diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics0.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics0.txt new file mode 100755 index 0000000..4830f61 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics0.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics1.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics1.txt new file mode 100755 index 0000000..4830f61 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics1.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics2.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics2.txt new file mode 100755 index 0000000..4830f61 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics2.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/config_nvdsanalytics.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/config_nvdsanalytics.txt new file mode 100755 index 0000000..ea5e5b2 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/config_nvdsanalytics.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=1 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/dstest5_msgconv_sample_config.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/dstest5_msgconv_sample_config.yml new file mode 100755 index 0000000..7b10f9c --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/dstest5_msgconv_sample_config.yml @@ -0,0 +1,119 @@ +############################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################# +sensor0: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor1: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor2: + enable: 1 + type: Camera + id: HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor3: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +place0: + enable: 1 + id: 0 + type: intersection/road + name: HWY_20_AND_LOCUST__EBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place1: + enable: 1 + id: 1 + type: intersection/road + name: HWY_20_AND_LOCUST__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place2: + enable: 1 + id: 2 + type: intersection/road + name: HWY_20_AND_DEVON__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place3: + enable: 1 + id: 3 + type: intersection/road + name: HWY_20_AND_LOCUST + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +analytics0: + enable: 1 + id: XYZ_1 + description: Vehicle Detection and License Plate Recognition + source: OpenALR + version: 1.0 + +analytics1: + enable: 1 + id: XYZ_2 + description: Vehicle Detection and License Plate Recognition 1 + source: OpenALR + version: 1.0 + +analytics2: + enable: 1 + id: XYZ_3 + description: Vehicle Detection and License Plate Recognition 2 + source: OpenALR + version: 1.0 + +analytics3: + enable: 1 + id: XYZ_4 + description: Vehicle Detection and License Plate Recognition 4 + source: OpenALR + version: 1.0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie0.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie0.yml new file mode 100755 index 0000000..c499a4e --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie0.yml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 11 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 12 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_carmake.txt diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie1.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie1.yml new file mode 100755 index 0000000..0d4f705 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie1.yml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 0 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 20 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie2.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie2.yml new file mode 100755 index 0000000..c0ec3b7 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie2.yml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 30 + operate-on-gie-id: 3 + operate-on-class-ids: 0 + config-file: ../../US_LPD/config_uslpd_inferserver.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 31 + operate-on-gie-id: 30 + operate-on-class-ids: 0 + config-file: ../../us_lprnet/config_uslpr_inferserver.txt + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/source4_1080p_dec_parallel_infer.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/source4_1080p_dec_parallel_infer.yml new file mode 100755 index 0000000..395de59 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/source4_1080p_dec_parallel_infer.yml @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#branch1 yolo +#brach2 bodypose +# +# +# +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 5 + ##gie-kitti-output-dir=streamscl + +tiled-display: + enable: 1 + rows: 2 + columns: 2 + width: 1280 + height: 720 + gpu-id: 0 + #(0): nvbuf-mem-default - Default memory allocated, specific to particular platform + #(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla + #(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla + #(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla + #(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson + nvbuf-memory-type: 0 + +source: + csv-file-path: sources_4.csv + #csv-file-path: sources_4_different_source.csv + #csv-file-path: sources_4_rtsp.csv + +sink0: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 7=nv3dsink (Jetson only) + type: 2 + sync: 1 + source-id: 0 + gpu-id: 0 + nvbuf-memory-type: 0 + +sink1: + enable: 1 + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 1 + #iframeinterval=10 + bitrate: 2000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + output-file: out.mp4 + source-id: 0 + +sink2: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming + type: 4 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 0 + bitrate: 4000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + # set below properties in case of RTSPStreaming + rtsp-port: 8554 + udp-port: 5400 + +sink3: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: ;; + topic: + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + +sink4: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-conv-msg2p-new-api: 0 + #Frame interval at which payload is generated + msg-conv-frame-interval: 30 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: 10.23.136.84;9092 + topic: dstest + disable-msgconv: 1 + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + + +# sink type = 6 by default creates msg converter + broker. +# To use multiple brokers use this group for converter and use +# sink type = 6 with disable-msgconv : 1 +message-converter: + enable: 0 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + # Name of library having custom implementation. + msg-conv-msg2p-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv.so + # Id of component in case only selected message to parse. + #msg-conv-comp-id: + +osd: + enable: 1 + gpu-id: 0 + border-width: 1 + text-size: 15 + #value changed + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Serif + show-clock: 0 + clock-x-offset: 800 + clock-y-offset: 820 + clock-text-size: 12 + clock-color: 1;0;0;0 + nvbuf-memory-type: 0 + +streammux: + gpu-id: 0 + ##Boolean property to inform muxer that sources are live + live-source: 0 + buffer-pool-size: 4 + batch-size: 4 + ##time out in usec, to wait after the first buffer is available + ##to push the batch even if the complete batch is not formed + batched-push-timeout: 400000 + ## Set muxer output width and height + width: 1920 + height: 1080 + ##Enable to maintain aspect ratio wrt source, and allow black borders, works + ##along with width, height properties + enable-padding: 0 + nvbuf-memory-type: 0 + +primary-gie0: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 0 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + #interval: 0 + gie-unique-id: 1 + nvbuf-memory-type: 0 + config-file: ../../yolov4/config_yolov4_infer.txt + +branch0: + ## pgie's id + pgie-id: 1 + ## select sources by sourceid + src-ids: 0;1;2 + +tracker0: + enable: 0 + cfg-file-path: tracker0.yml + +nvds-analytics0: + enable: 0 + cfg-file-path: analytics0.txt + +secondary-gie0: + enable: 0 + ##support mulptiple sgie. + cfg-file-path: secondary-gie0.yml + +primary-gie1: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 0 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 2 + nvbuf-memory-type: 0 + #config-file: ../../bodypose2d/config_body2_inferserver.txt + config-file: ../../bodypose2d/config_body2_infer.txt + +branch1: + ## pgie's id + pgie-id: 2 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker1: + enable: 0 + cfg-file-path: tracker1.yml + +nvds-analytics1: + enable: 0 + cfg-file-path: analytics1.txt + +secondary-gie1: + enable: 0 + ##supoort multiple sgie + cfg-file-path: secondary-gie1.yml + +branch2: + ## pgie's id + pgie-id: 3 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker2: + enable: 1 + cfg-file-path: tracker2.yml + +nvds-analytics2: + enable: 0 + cfg-file-path: analytics2.txt + +secondary-gie2: + enable: 1 + ##supoort multiple sgie + cfg-file-path: secondary-gie2.yml + + +meta-mux: + enable: 1 + config-file: ../../metamux/config_metamux0.txt + + +tests: + file-loop: 0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4.csv new file mode 100755 index 0000000..213c663 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4.csv @@ -0,0 +1,2 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,4,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4_different_source.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4_different_source.csv new file mode 100755 index 0000000..29a2b48 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4_different_source.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,1,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4_rtsp.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4_rtsp.csv new file mode 100755 index 0000000..2663e40 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4_rtsp.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 +1,4,rtsp://10.19.225.227/media/video1,1,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker0.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker0.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker0.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker1.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker1.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker1.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker2.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker2.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker2.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics0.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics0.txt new file mode 100755 index 0000000..ea5e5b2 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics0.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=1 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics1.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics1.txt new file mode 100755 index 0000000..ea5e5b2 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics1.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=1 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics2.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics2.txt new file mode 100755 index 0000000..4830f61 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics2.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/config_nvdsanalytics.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/config_nvdsanalytics.txt new file mode 100755 index 0000000..ea5e5b2 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/config_nvdsanalytics.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=1 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/dstest5_msgconv_sample_config.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/dstest5_msgconv_sample_config.yml new file mode 100755 index 0000000..3004124 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/dstest5_msgconv_sample_config.yml @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sensor0: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor1: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor2: + enable: 1 + type: Camera + id: HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor3: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +place0: + enable: 1 + id: 0 + type: intersection/road + name: HWY_20_AND_LOCUST__EBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place1: + enable: 1 + id: 1 + type: intersection/road + name: HWY_20_AND_LOCUST__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place2: + enable: 1 + id: 2 + type: intersection/road + name: HWY_20_AND_DEVON__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place3: + enable: 1 + id: 3 + type: intersection/road + name: HWY_20_AND_LOCUST + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +analytics0: + enable: 1 + id: XYZ_1 + description: Vehicle Detection and License Plate Recognition + source: OpenALR + version: 1.0 + +analytics1: + enable: 1 + id: XYZ_2 + description: Vehicle Detection and License Plate Recognition 1 + source: OpenALR + version: 1.0 + +analytics2: + enable: 1 + id: XYZ_3 + description: Vehicle Detection and License Plate Recognition 2 + source: OpenALR + version: 1.0 + +analytics3: + enable: 1 + id: XYZ_4 + description: Vehicle Detection and License Plate Recognition 4 + source: OpenALR + version: 1.0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie0.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie0.yml new file mode 100755 index 0000000..a05c300 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie0.yml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 11 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie1.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie1.yml new file mode 100755 index 0000000..0d4f705 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie1.yml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 0 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 20 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie2.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie2.yml new file mode 100755 index 0000000..c0ec3b7 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie2.yml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 30 + operate-on-gie-id: 3 + operate-on-class-ids: 0 + config-file: ../../US_LPD/config_uslpd_inferserver.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 31 + operate-on-gie-id: 30 + operate-on-class-ids: 0 + config-file: ../../us_lprnet/config_uslpr_inferserver.txt + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/source4_1080p_dec_parallel_infer.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/source4_1080p_dec_parallel_infer.yml new file mode 100755 index 0000000..bc777f1 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/source4_1080p_dec_parallel_infer.yml @@ -0,0 +1,288 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 5 + ##gie-kitti-output-dir=streamscl + +tiled-display: + enable: 1 + rows: 2 + columns: 2 + width: 1280 + height: 720 + gpu-id: 0 + #(0): nvbuf-mem-default - Default memory allocated, specific to particular platform + #(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla + #(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla + #(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla + #(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson + nvbuf-memory-type: 0 + +source: + #csv-file-path: sources_4.csv + csv-file-path: sources_4_different_source.csv + #csv-file-path: sources_4_rtsp.csv + +sink0: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 7=nv3dsink (Jetson only) + type: 2 + sync: 1 + source-id: 0 + gpu-id: 0 + nvbuf-memory-type: 0 + +sink1: + enable: 1 + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 1 + #iframeinterval=10 + bitrate: 2000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + output-file: out.mp4 + source-id: 0 + +sink2: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming + type: 4 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 0 + bitrate: 4000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + # set below properties in case of RTSPStreaming + rtsp-port: 8554 + udp-port: 5400 + +sink3: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: ;; + topic: + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + +sink4: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-conv-msg2p-new-api: 0 + #Frame interval at which payload is generated + msg-conv-frame-interval: 30 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: 10.23.136.84;9092 + topic: dstest + disable-msgconv: 1 + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + + +# sink type = 6 by default creates msg converter + broker. +# To use multiple brokers use this group for converter and use +# sink type = 6 with disable-msgconv : 1 +message-converter: + enable: 0 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + # Name of library having custom implementation. + msg-conv-msg2p-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv.so + # Id of component in case only selected message to parse. + #msg-conv-comp-id: + +osd: + enable: 1 + gpu-id: 0 + border-width: 1 + text-size: 15 + #value changed + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Serif + show-clock: 0 + clock-x-offset: 800 + clock-y-offset: 820 + clock-text-size: 12 + clock-color: 1;0;0;0 + nvbuf-memory-type: 0 + +streammux: + gpu-id: 0 + ##Boolean property to inform muxer that sources are live + live-source: 0 + buffer-pool-size: 4 + batch-size: 4 + ##time out in usec, to wait after the first buffer is available + ##to push the batch even if the complete batch is not formed + batched-push-timeout: 400000 + ## Set muxer output width and height + width: 1920 + height: 1080 + ##Enable to maintain aspect ratio wrt source, and allow black borders, works + ##along with width, height properties + enable-padding: 0 + nvbuf-memory-type: 0 + +primary-gie0: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 0 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + #interval: 0 + gie-unique-id: 1 + nvbuf-memory-type: 0 + config-file: ../../yolov4/config_yolov4_infer.txt + +branch0: + ## pgie's id + pgie-id: 1 + ## select sources by sourceid + src-ids: 0;1;2 + +tracker0: + enable: 0 + cfg-file-path: tracker0.yml + +nvds-analytics0: + enable: 0 + cfg-file-path: analytics0.txt + +secondary-gie0: + enable: 0 + ##support mulptiple sgie. + cfg-file-path: secondary-gie0.yml + +primary-gie1: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 1 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 2 + nvbuf-memory-type: 0 + #config-file: ../../bodypose2d/config_body2_infer.txt + config-file: ../../bodypose2d/config_body2_inferserver.txt + +branch1: + ## pgie's id + pgie-id: 2 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker1: + enable: 0 + cfg-file-path: tracker1.yml + +nvds-analytics1: + enable: 0 + cfg-file-path: analytics1.txt + +secondary-gie1: + enable: 0 + ##supoort multiple sgie + cfg-file-path: secondary-gie1.yml + +primary-gie2: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 1 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 3 + nvbuf-memory-type: 0 + config-file: ../../trafficcamnet/config_trafficcamnet_inferserver.txt + +branch2: + ## pgie's id + pgie-id: 3 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker2: + enable: 1 + cfg-file-path: tracker2.yml + +nvds-analytics2: + enable: 1 + cfg-file-path: analytics2.txt + +secondary-gie2: + enable: 1 + ##supoort multiple sgie + cfg-file-path: secondary-gie2.yml + + +meta-mux: + enable: 1 + config-file: ../../metamux/config_metamux0.txt + + +tests: + file-loop: 0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4.csv new file mode 100755 index 0000000..213c663 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4.csv @@ -0,0 +1,2 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,4,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4_different_source.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4_different_source.csv new file mode 100755 index 0000000..f343e40 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4_different_source.csv @@ -0,0 +1,3 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h265.mp4,2,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4_rtsp.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4_rtsp.csv new file mode 100755 index 0000000..2663e40 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4_rtsp.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 +1,4,rtsp://10.19.225.227/media/video1,1,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker0.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker0.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker0.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker1.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker1.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker1.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker2.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker2.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker2.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics0.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics0.txt new file mode 100755 index 0000000..4830f61 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics0.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics1.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics1.txt new file mode 100755 index 0000000..4830f61 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics1.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics2.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics2.txt new file mode 100755 index 0000000..4830f61 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics2.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/config_nvdsanalytics.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/config_nvdsanalytics.txt new file mode 100755 index 0000000..ea5e5b2 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/config_nvdsanalytics.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=1 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/dstest5_msgconv_sample_config.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/dstest5_msgconv_sample_config.yml new file mode 100755 index 0000000..3004124 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/dstest5_msgconv_sample_config.yml @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sensor0: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor1: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor2: + enable: 1 + type: Camera + id: HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor3: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +place0: + enable: 1 + id: 0 + type: intersection/road + name: HWY_20_AND_LOCUST__EBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place1: + enable: 1 + id: 1 + type: intersection/road + name: HWY_20_AND_LOCUST__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place2: + enable: 1 + id: 2 + type: intersection/road + name: HWY_20_AND_DEVON__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place3: + enable: 1 + id: 3 + type: intersection/road + name: HWY_20_AND_LOCUST + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +analytics0: + enable: 1 + id: XYZ_1 + description: Vehicle Detection and License Plate Recognition + source: OpenALR + version: 1.0 + +analytics1: + enable: 1 + id: XYZ_2 + description: Vehicle Detection and License Plate Recognition 1 + source: OpenALR + version: 1.0 + +analytics2: + enable: 1 + id: XYZ_3 + description: Vehicle Detection and License Plate Recognition 2 + source: OpenALR + version: 1.0 + +analytics3: + enable: 1 + id: XYZ_4 + description: Vehicle Detection and License Plate Recognition 4 + source: OpenALR + version: 1.0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie0.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie0.yml new file mode 100755 index 0000000..c499a4e --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie0.yml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 11 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 12 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_carmake.txt diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie1.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie1.yml new file mode 100755 index 0000000..2157b0f --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie1.yml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 0 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 20 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie2.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie2.yml new file mode 100755 index 0000000..c0ec3b7 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie2.yml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 30 + operate-on-gie-id: 3 + operate-on-class-ids: 0 + config-file: ../../US_LPD/config_uslpd_inferserver.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 31 + operate-on-gie-id: 30 + operate-on-class-ids: 0 + config-file: ../../us_lprnet/config_uslpr_inferserver.txt + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/source4_1080p_dec_parallel_infer.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/source4_1080p_dec_parallel_infer.yml new file mode 100755 index 0000000..b1dbc79 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/source4_1080p_dec_parallel_infer.yml @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#branch1 yolo +#brach2 bodypose +# +# +# +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 5 + ##gie-kitti-output-dir=streamscl + +tiled-display: + enable: 1 + rows: 2 + columns: 2 + width: 1280 + height: 720 + gpu-id: 0 + #(0): nvbuf-mem-default - Default memory allocated, specific to particular platform + #(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla + #(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla + #(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla + #(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson + nvbuf-memory-type: 0 + #which source should be showed, -1 means showing all. + show-source: 2 + +source: + csv-file-path: sources_4.csv + #csv-file-path: sources_4_different_source.csv + #csv-file-path: sources_4_rtsp.csv + +sink0: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 7=nv3dsink (Jetson only) + type: 2 + sync: 1 + source-id: 0 + gpu-id: 0 + nvbuf-memory-type: 0 + +sink1: + enable: 1 + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 1 + #iframeinterval=10 + bitrate: 2000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + output-file: out.mp4 + source-id: 0 + +sink2: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming + type: 4 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 0 + bitrate: 4000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + # set below properties in case of RTSPStreaming + rtsp-port: 8554 + udp-port: 5400 + +sink3: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: ;; + topic: + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + +sink4: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-conv-msg2p-new-api: 0 + #Frame interval at which payload is generated + msg-conv-frame-interval: 30 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: 10.23.136.84;9092 + topic: dstest + disable-msgconv: 1 + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + + +# sink type = 6 by default creates msg converter + broker. +# To use multiple brokers use this group for converter and use +# sink type = 6 with disable-msgconv : 1 +message-converter: + enable: 0 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + # Name of library having custom implementation. + msg-conv-msg2p-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv.so + # Id of component in case only selected message to parse. + #msg-conv-comp-id: + +osd: + enable: 1 + gpu-id: 0 + border-width: 1 + text-size: 15 + #value changed + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Serif + show-clock: 0 + clock-x-offset: 800 + clock-y-offset: 820 + clock-text-size: 12 + clock-color: 1;0;0;0 + nvbuf-memory-type: 0 + +streammux: + gpu-id: 0 + ##Boolean property to inform muxer that sources are live + live-source: 0 + buffer-pool-size: 4 + batch-size: 4 + ##time out in usec, to wait after the first buffer is available + ##to push the batch even if the complete batch is not formed + batched-push-timeout: 400000 + ## Set muxer output width and height + width: 1920 + height: 1080 + ##Enable to maintain aspect ratio wrt source, and allow black borders, works + ##along with width, height properties + enable-padding: 0 + nvbuf-memory-type: 0 + +primary-gie0: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 0 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + #interval: 0 + gie-unique-id: 1 + nvbuf-memory-type: 0 + config-file: ../../yolov4/config_yolov4_infer.txt + +branch0: + ## pgie's id + pgie-id: 1 + ## select sources by sourceid + src-ids: 0;1;2 + +tracker0: + enable: 0 + cfg-file-path: tracker0.yml + +nvds-analytics0: + enable: 0 + cfg-file-path: analytics0.txt + +secondary-gie0: + enable: 0 + ##support mulptiple sgie. + cfg-file-path: secondary-gie0.yml + +primary-gie1: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 0 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 2 + nvbuf-memory-type: 0 + #config-file: ../../bodypose2d/config_body2_inferserver.txt + config-file: ../../bodypose2d/config_body2_infer.txt + +branch1: + ## pgie's id + pgie-id: 2 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker1: + enable: 0 + cfg-file-path: tracker1.yml + +nvds-analytics1: + enable: 0 + cfg-file-path: analytics1.txt + +secondary-gie1: + enable: 0 + ##supoort multiple sgie + cfg-file-path: secondary-gie1.yml + +meta-mux: + enable: 1 + config-file: ../../metamux/config_metamux0.txt + + +tests: + file-loop: 0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4.csv new file mode 100755 index 0000000..213c663 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4.csv @@ -0,0 +1,2 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,4,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4_different_source.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4_different_source.csv new file mode 100755 index 0000000..29a2b48 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4_different_source.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,1,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4_rtsp.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4_rtsp.csv new file mode 100755 index 0000000..2663e40 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4_rtsp.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 +1,4,rtsp://10.19.225.227/media/video1,1,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker0.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker0.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker0.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker1.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker1.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker1.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker2.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker2.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker2.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics0.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics0.txt new file mode 100755 index 0000000..4830f61 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics0.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics1.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics1.txt new file mode 100755 index 0000000..4830f61 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics1.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics2.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics2.txt new file mode 100755 index 0000000..4830f61 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics2.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/config_nvdsanalytics.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/config_nvdsanalytics.txt new file mode 100755 index 0000000..ea5e5b2 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/config_nvdsanalytics.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=1 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/dstest5_msgconv_sample_config.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/dstest5_msgconv_sample_config.yml new file mode 100755 index 0000000..3004124 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/dstest5_msgconv_sample_config.yml @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sensor0: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor1: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor2: + enable: 1 + type: Camera + id: HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor3: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +place0: + enable: 1 + id: 0 + type: intersection/road + name: HWY_20_AND_LOCUST__EBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place1: + enable: 1 + id: 1 + type: intersection/road + name: HWY_20_AND_LOCUST__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place2: + enable: 1 + id: 2 + type: intersection/road + name: HWY_20_AND_DEVON__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place3: + enable: 1 + id: 3 + type: intersection/road + name: HWY_20_AND_LOCUST + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +analytics0: + enable: 1 + id: XYZ_1 + description: Vehicle Detection and License Plate Recognition + source: OpenALR + version: 1.0 + +analytics1: + enable: 1 + id: XYZ_2 + description: Vehicle Detection and License Plate Recognition 1 + source: OpenALR + version: 1.0 + +analytics2: + enable: 1 + id: XYZ_3 + description: Vehicle Detection and License Plate Recognition 2 + source: OpenALR + version: 1.0 + +analytics3: + enable: 1 + id: XYZ_4 + description: Vehicle Detection and License Plate Recognition 4 + source: OpenALR + version: 1.0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie0.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie0.yml new file mode 100755 index 0000000..9f33ab5 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie0.yml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 0 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 11 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_secondary_vehicletypes.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 0 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 12 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_secondary_vehiclemake.txt + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie1.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie1.yml new file mode 100755 index 0000000..2157b0f --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie1.yml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 0 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 20 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie2.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie2.yml new file mode 100755 index 0000000..c0ec3b7 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie2.yml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 30 + operate-on-gie-id: 3 + operate-on-class-ids: 0 + config-file: ../../US_LPD/config_uslpd_inferserver.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 31 + operate-on-gie-id: 30 + operate-on-class-ids: 0 + config-file: ../../us_lprnet/config_uslpr_inferserver.txt + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/source4_1080p_dec_parallel_infer.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/source4_1080p_dec_parallel_infer.yml new file mode 100755 index 0000000..ba87310 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/source4_1080p_dec_parallel_infer.yml @@ -0,0 +1,292 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#branch1 vehicle detecor + color checing + maker checking + type checking +#brach2 peoplenet + tracker + analytics +#branch3 trafficnet + tracker + lpd + lpr +# +# +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 5 + ##gie-kitti-output-dir=streamscl + +tiled-display: + enable: 1 + rows: 2 + columns: 2 + width: 1280 + height: 720 + gpu-id: 0 + #(0): nvbuf-mem-default - Default memory allocated, specific to particular platform + #(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla + #(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla + #(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla + #(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson + nvbuf-memory-type: 0 + +source: + csv-file-path: sources_4.csv + #csv-file-path: sources_4_different_source.csv + #csv-file-path: sources_4_rtsp.csv + +sink0: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 7=nv3dsink (Jetson only) + type: 2 + sync: 1 + source-id: 0 + gpu-id: 0 + nvbuf-memory-type: 0 + +sink1: + enable: 1 + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 1 + #iframeinterval=10 + bitrate: 2000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + output-file: out.mp4 + source-id: 0 + +sink2: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming + type: 4 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 0 + bitrate: 4000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + # set below properties in case of RTSPStreaming + rtsp-port: 8554 + udp-port: 5400 + +sink3: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: ;; + topic: + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + +sink4: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-conv-msg2p-new-api: 0 + #Frame interval at which payload is generated + msg-conv-frame-interval: 30 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: 10.23.136.84;9092 + topic: dstest + disable-msgconv: 1 + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + + +# sink type = 6 by default creates msg converter + broker. +# To use multiple brokers use this group for converter and use +# sink type = 6 with disable-msgconv : 1 +message-converter: + enable: 0 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + # Name of library having custom implementation. + msg-conv-msg2p-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv.so + # Id of component in case only selected message to parse. + #msg-conv-comp-id: + +osd: + enable: 1 + gpu-id: 0 + border-width: 1 + text-size: 15 + #value changed + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Serif + show-clock: 0 + clock-x-offset: 800 + clock-y-offset: 820 + clock-text-size: 12 + clock-color: 1;0;0;0 + nvbuf-memory-type: 0 + +streammux: + gpu-id: 0 + ##Boolean property to inform muxer that sources are live + live-source: 0 + buffer-pool-size: 4 + batch-size: 4 + ##time out in usec, to wait after the first buffer is available + ##to push the batch even if the complete batch is not formed + batched-push-timeout: 40000 + ## Set muxer output width and height + width: 1920 + height: 1080 + ##Enable to maintain aspect ratio wrt source, and allow black borders, works + ##along with width, height properties + enable-padding: 0 + nvbuf-memory-type: 0 + +primary-gie0: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 0 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 30 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + #interval: 0 + gie-unique-id: 1 + nvbuf-memory-type: 0 + config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_primary.txt + +branch0: + ## pgie's id + pgie-id: 1 + ## select sources by sourceid + src-ids: 0;1;2 + +tracker0: + enable: 1 + cfg-file-path: tracker0.yml + +nvds-analytics0: + enable: 0 + cfg-file-path: analytics0.txt + +secondary-gie0: + enable: 1 + ##support mulptiple sgie. + cfg-file-path: secondary-gie0.yml + +primary-gie1: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 1 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 2 + nvbuf-memory-type: 0 + config-file: ../../peoplenet/config_peoplenet_inferserver.txt + +branch1: + ## pgie's id + pgie-id: 2 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker1: + enable: 1 + cfg-file-path: tracker1.yml + +nvds-analytics1: + enable: 1 + cfg-file-path: analytics1.txt + +secondary-gie1: + enable: 0 + ##supoort multiple sgie + cfg-file-path: secondary-gie1.yml + +primary-gie2: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 1 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 3 + nvbuf-memory-type: 0 + config-file: ../../trafficcamnet/config_trafficcamnet_inferserver.txt + +branch2: + ## pgie's id + pgie-id: 3 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker2: + enable: 1 + cfg-file-path: tracker2.yml + +nvds-analytics2: + enable: 0 + cfg-file-path: analytics2.txt + +secondary-gie2: + enable: 1 + ##supoort multiple sgie + cfg-file-path: secondary-gie2.yml + + +meta-mux: + enable: 1 + config-file: ../../metamux/config_metamux1.txt + + +tests: + file-loop: 0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4.csv new file mode 100755 index 0000000..213c663 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4.csv @@ -0,0 +1,2 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,4,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4_different_source.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4_different_source.csv new file mode 100755 index 0000000..29a2b48 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4_different_source.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,1,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4_rtsp.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4_rtsp.csv new file mode 100755 index 0000000..2663e40 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4_rtsp.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 +1,4,rtsp://10.19.225.227/media/video1,1,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker0.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker0.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker0.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker1.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker1.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker1.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker2.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker2.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker2.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics0.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics0.txt new file mode 100755 index 0000000..4830f61 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics0.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics1.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics1.txt new file mode 100755 index 0000000..4830f61 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics1.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics2.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics2.txt new file mode 100755 index 0000000..4830f61 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics2.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/config_nvdsanalytics.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/config_nvdsanalytics.txt new file mode 100755 index 0000000..ea5e5b2 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/config_nvdsanalytics.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=1 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/dstest5_msgconv_sample_config.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/dstest5_msgconv_sample_config.yml new file mode 100755 index 0000000..3004124 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/dstest5_msgconv_sample_config.yml @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sensor0: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor1: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor2: + enable: 1 + type: Camera + id: HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor3: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +place0: + enable: 1 + id: 0 + type: intersection/road + name: HWY_20_AND_LOCUST__EBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place1: + enable: 1 + id: 1 + type: intersection/road + name: HWY_20_AND_LOCUST__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place2: + enable: 1 + id: 2 + type: intersection/road + name: HWY_20_AND_DEVON__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place3: + enable: 1 + id: 3 + type: intersection/road + name: HWY_20_AND_LOCUST + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +analytics0: + enable: 1 + id: XYZ_1 + description: Vehicle Detection and License Plate Recognition + source: OpenALR + version: 1.0 + +analytics1: + enable: 1 + id: XYZ_2 + description: Vehicle Detection and License Plate Recognition 1 + source: OpenALR + version: 1.0 + +analytics2: + enable: 1 + id: XYZ_3 + description: Vehicle Detection and License Plate Recognition 2 + source: OpenALR + version: 1.0 + +analytics3: + enable: 1 + id: XYZ_4 + description: Vehicle Detection and License Plate Recognition 4 + source: OpenALR + version: 1.0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie0.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie0.yml new file mode 100755 index 0000000..c499a4e --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie0.yml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 11 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 12 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_carmake.txt diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie1.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie1.yml new file mode 100755 index 0000000..2157b0f --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie1.yml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 0 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 20 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie2.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie2.yml new file mode 100755 index 0000000..c0ec3b7 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie2.yml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 30 + operate-on-gie-id: 3 + operate-on-class-ids: 0 + config-file: ../../US_LPD/config_uslpd_inferserver.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 31 + operate-on-gie-id: 30 + operate-on-class-ids: 0 + config-file: ../../us_lprnet/config_uslpr_inferserver.txt + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/source4_1080p_dec_parallel_infer.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/source4_1080p_dec_parallel_infer.yml new file mode 100755 index 0000000..07b1c22 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/source4_1080p_dec_parallel_infer.yml @@ -0,0 +1,292 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#branch1 vehicle detecor + color checing + maker checking + type checking +#brach2 peoplenet + tracker + analytics +#branch3 trafficnet + tracker + lpd + lpr +# +# +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 5 + ##gie-kitti-output-dir=streamscl + +tiled-display: + enable: 1 + rows: 2 + columns: 2 + width: 1280 + height: 720 + gpu-id: 0 + #(0): nvbuf-mem-default - Default memory allocated, specific to particular platform + #(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla + #(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla + #(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla + #(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson + nvbuf-memory-type: 0 + +source: + csv-file-path: sources_4.csv + #csv-file-path: sources_4_different_source.csv + #csv-file-path: sources_4_rtsp.csv + +sink0: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 7=nv3dsink (Jetson only) + type: 2 + sync: 1 + source-id: 0 + gpu-id: 0 + nvbuf-memory-type: 0 + +sink1: + enable: 1 + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 1 + #iframeinterval=10 + bitrate: 2000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + output-file: out.mp4 + source-id: 0 + +sink2: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming + type: 4 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 0 + bitrate: 4000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + # set below properties in case of RTSPStreaming + rtsp-port: 8554 + udp-port: 5400 + +sink3: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: ;; + topic: + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + +sink4: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-conv-msg2p-new-api: 0 + #Frame interval at which payload is generated + msg-conv-frame-interval: 30 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: 10.23.136.84;9092 + topic: dstest + disable-msgconv: 1 + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + + +# sink type = 6 by default creates msg converter + broker. +# To use multiple brokers use this group for converter and use +# sink type = 6 with disable-msgconv : 1 +message-converter: + enable: 0 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + # Name of library having custom implementation. + msg-conv-msg2p-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv.so + # Id of component in case only selected message to parse. + #msg-conv-comp-id: + +osd: + enable: 1 + gpu-id: 0 + border-width: 1 + text-size: 15 + #value changed + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Serif + show-clock: 0 + clock-x-offset: 800 + clock-y-offset: 820 + clock-text-size: 12 + clock-color: 1;0;0;0 + nvbuf-memory-type: 0 + +streammux: + gpu-id: 0 + ##Boolean property to inform muxer that sources are live + live-source: 0 + buffer-pool-size: 4 + batch-size: 4 + ##time out in usec, to wait after the first buffer is available + ##to push the batch even if the complete batch is not formed + batched-push-timeout: 40000 + ## Set muxer output width and height + width: 1920 + height: 1080 + ##Enable to maintain aspect ratio wrt source, and allow black borders, works + ##along with width, height properties + enable-padding: 0 + nvbuf-memory-type: 0 + +primary-gie0: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 1 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + #interval: 0 + gie-unique-id: 1 + nvbuf-memory-type: 0 + config-file: ../../trafficcamnet/config_trafficcamnet_inferserver.txt + +branch0: + ## pgie's id + pgie-id: 1 + ## select sources by sourceid + src-ids: 0;1;2 + +tracker0: + enable: 0 + cfg-file-path: tracker0.yml + +nvds-analytics0: + enable: 0 + cfg-file-path: analytics0.txt + +secondary-gie0: + enable: 1 + ##support mulptiple sgie. + cfg-file-path: secondary-gie0.yml + +primary-gie1: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 1 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 2 + nvbuf-memory-type: 0 + config-file: ../../peoplenet/config_peoplenet_inferserver.txt + +branch1: + ## pgie's id + pgie-id: 2 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker1: + enable: 1 + cfg-file-path: tracker1.yml + +nvds-analytics1: + enable: 1 + cfg-file-path: analytics1.txt + +secondary-gie1: + enable: 0 + ##supoort multiple sgie + cfg-file-path: secondary-gie1.yml + +primary-gie2: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 1 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 3 + nvbuf-memory-type: 0 + config-file: ../../trafficcamnet/config_trafficcamnet_inferserver.txt + +branch2: + ## pgie's id + pgie-id: 3 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker2: + enable: 1 + cfg-file-path: tracker2.yml + +nvds-analytics2: + enable: 0 + cfg-file-path: analytics2.txt + +secondary-gie2: + enable: 1 + ##supoort multiple sgie + cfg-file-path: secondary-gie2.yml + + +meta-mux: + enable: 1 + config-file: ../../metamux/config_metamux1.txt + + +tests: + file-loop: 0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4.csv new file mode 100755 index 0000000..213c663 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4.csv @@ -0,0 +1,2 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,4,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4_different_source.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4_different_source.csv new file mode 100755 index 0000000..29a2b48 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4_different_source.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,1,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4_rtsp.csv b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4_rtsp.csv new file mode 100755 index 0000000..2663e40 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4_rtsp.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 +1,4,rtsp://10.19.225.227/media/video1,1,0,0 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker0.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker0.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker0.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker1.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker1.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker1.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker2.yml b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker2.yml new file mode 100755 index 0000000..73756a6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker2.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/bodypose2d/config_body2_infer.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/bodypose2d/config_body2_infer.txt new file mode 100755 index 0000000..8a2745b --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/bodypose2d/config_body2_infer.txt @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +gpu-id=0 +model-engine-file=../../../../tritonserver/models/bodypose2d/1/model.onnx_b4_gpu0_fp16.engine +onnx-file=../../../../tritonserver/models/bodypose2d/1/model.onnx +batch-size=4 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=3 +gie-unique-id=2 +#output-blob-names=output_bbox/BiasAdd;output_cov/Sigmoid +#0=Detection 1=Classifier 2=Segmentation +network-type=100 +process-mode=1 +net-scale-factor=0.01743071291615827 +offsets=114.74;114.74;114.74 +#0=RGB 1=BGR 2=GRAY +model-color-format=0 +output-tensor-meta=1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/bodypose2d/config_body2_inferserver.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/bodypose2d/config_body2_inferserver.txt new file mode 100755 index 0000000..aa00751 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/bodypose2d/config_body2_inferserver.txt @@ -0,0 +1,61 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 2 + gpu_ids: [0] + max_batch_size: 4 + backend { + triton { + model_name: "bodypose2d" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.01743071291615827 + channel_offsets: [114.74,114.74,114.74] + } + } + + postprocess { + other {} + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + interval: 0 +} + +output_control { + output_tensor_meta: true +} diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/metamux/config_metamux0.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/metamux/config_metamux0.txt new file mode 100755 index 0000000..bc8449b --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/metamux/config_metamux0.txt @@ -0,0 +1,35 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +enable=1 +# sink pad name which data will be pass to src pad. +active-pad=sink_0 +# default pts-tolerance is 60 ms. +pts-tolerance=60000 + +[user-configs] + +[group-0] +# src-ids-model-= +# mux all source if don't set it. +src-ids-model-1=0;1 +src-ids-model-2=1;2 +src-ids-model-3=1;2 \ No newline at end of file diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/metamux/config_metamux1.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/metamux/config_metamux1.txt new file mode 100755 index 0000000..3996397 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/metamux/config_metamux1.txt @@ -0,0 +1,35 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +enable=1 +# sink pad name which data will be pass to src pad. +active-pad=sink_0 +# default pts-tolerance is 60 ms. +pts-tolerance=60000 + +[user-configs] + +[group-0] +# src-ids-model-= +# mux all source if don't set it. +src-ids-model-1=0;1 +src-ids-model-2=1;2 +src-ids-model-3=0;1 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/peoplenet/config_peoplenet_inferserver.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/peoplenet/config_peoplenet_inferserver.txt new file mode 100755 index 0000000..5289256 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/peoplenet/config_peoplenet_inferserver.txt @@ -0,0 +1,70 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 3 + gpu_ids: [0] + max_batch_size: 8 + backend { + triton { + model_name: "peoplenet" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + channel_offsets: [0.0,0.0,0.0] + } + } + + postprocess { + labelfile_path: "../../../../tritonserver/models/peoplenet/labels.txt" + detection { + num_detected_classes: 4 + + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 6 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + interval: 0 +} diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/trafficcamnet/config_trafficcamnet_inferserver.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/trafficcamnet/config_trafficcamnet_inferserver.txt new file mode 100755 index 0000000..02f0868 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/trafficcamnet/config_trafficcamnet_inferserver.txt @@ -0,0 +1,70 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 3 + gpu_ids: [0] + max_batch_size: 8 + backend { + triton { + model_name: "trafficcamnet" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + channel_offsets: [0.0,0.0,0.0] + } + } + + postprocess { + labelfile_path: "../../../../tritonserver/models/trafficcamnet/labels.txt" + detection { + num_detected_classes: 4 + + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 6 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + interval: 0 +} diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/us_lprnet/config_uslpr_inferserver.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/us_lprnet/config_uslpr_inferserver.txt new file mode 100755 index 0000000..01ea8b1 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/us_lprnet/config_uslpr_inferserver.txt @@ -0,0 +1,64 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 5 + gpu_ids: [0] + max_batch_size: 16 + backend { + triton { + model_name: "us_lprnet" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + channel_offsets: [0.0,0.0,0.0] + } + } + + postprocess { + classification { + custom_parse_classifier_func: "NvDsInferParseCustomNVPlate" + threshold: 0.5 + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 6 + } + custom_lib { + path: "../../gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser/libnvdsinfer_custom_impl_lpr.so" + } +} +input_control { + operate_on_gie_id: 30 + process_mode: PROCESS_MODE_CLIP_OBJECTS + interval: 0 +} diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_plan_engine_primary.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_plan_engine_primary.txt new file mode 100755 index 0000000..a6501fb --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_plan_engine_primary.txt @@ -0,0 +1,78 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 30 + backend { + inputs: [ { + name: "input_1" + }] + outputs: [ + {name: "conv2d_bbox"}, + {name: "conv2d_cov/Sigmoid"} + ] + triton { + model_name: "Primary_Detector" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: MEDIA_FORMAT_NONE + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + channel_offsets: [0, 0, 0] + } + } + + postprocess { + labelfile_path: "/opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/labels.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_secondary_plan_engine_carmake.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_secondary_plan_engine_carmake.txt new file mode 100755 index 0000000..86c6e40 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_secondary_plan_engine_carmake.txt @@ -0,0 +1,76 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 6 + gpu_ids: [0] + max_batch_size: 16 + backend { + inputs: [ { + name: "input_1:0" + }] + outputs: [ + {name: "predictions/Softmax:0"} + ] + triton { + model_name: "Secondary_CarMake" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 1.0 + channel_offsets: [124, 117, 104] + } + } + + postprocess { + labelfile_path: "../../../../tritonserver/models/Secondary_CarMake/labels.txt" + classification { + threshold: 0.51 + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + operate_on_class_ids: [0] + interval: 0 + async_mode: true + object_control { + bbox_filter { + min_width: 14 + min_height: 14 + } + } +} diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_secondary_plan_engine_vehicletypes.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_secondary_plan_engine_vehicletypes.txt new file mode 100755 index 0000000..7e08023 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_secondary_plan_engine_vehicletypes.txt @@ -0,0 +1,66 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 4 + gpu_ids: [0] + max_batch_size: 16 + backend { + triton { + model_name: "Secondary_VehicleTypes" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 1.0 + channel_offsets: [124, 117, 104] + } + } + + postprocess { + labelfile_path: "../../../../tritonserver/models/Secondary_VehicleTypes/labels.txt" + classification { + threshold: 0.2 + } + } + +} +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + operate_on_class_ids: [0] + interval: 0 + async_mode: true + object_control { + bbox_filter { + min_width: 14 + min_height: 14 + } + } +} diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/yolov4/coco.names b/deepstream_parallel_inference_app/tritonclient/sample/configs/yolov4/coco.names new file mode 100755 index 0000000..d70398f --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/yolov4/coco.names @@ -0,0 +1,80 @@ +person +bicycle +car +motorbike +aeroplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +sofa +potted plant +bed +dining table +toilet +tvmonitor +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush \ No newline at end of file diff --git a/deepstream_parallel_inference_app/tritonclient/sample/configs/yolov4/config_yolov4_infer.txt b/deepstream_parallel_inference_app/tritonclient/sample/configs/yolov4/config_yolov4_infer.txt new file mode 100755 index 0000000..3aaf797 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/configs/yolov4/config_yolov4_infer.txt @@ -0,0 +1,80 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8), model-file-format +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +#0=RGB, 1=BGR +model-color-format=0 +onnx-file=../../../../tritonserver/models/yolov4/1/yolov4_-1_3_416_416_dynamic.onnx.nms.onnx +model-engine-file=../../../../tritonserver/models/yolov4/1/yolov4_-1_3_416_416_dynamic.onnx.nms.onnx_b4_gpu0_fp16.engine +labelfile-path=coco.names +batch-size=1 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=80 +gie-unique-id=1 +network-type=0 +is-classifier=0 +## 0=Group Rectangles, 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +cluster-mode=2 +maintain-aspect-ratio=1 +parse-bbox-func-name=NvDsInferParseCustomYoloV4 +custom-lib-path=../../gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo/libnvdsinfer_custom_impl_Yolo.so +#scaling-filter=0 +#scaling-compute-hw=0 + +[class-attrs-all] +nms-iou-threshold=0.6 +pre-cluster-threshold=0.4 diff --git a/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvdsmetamux/README b/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvdsmetamux/README new file mode 100755 index 0000000..c07149c --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvdsmetamux/README @@ -0,0 +1,99 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +Refer to the DeepStream SDK documentation for a description of the plugin. +-------------------------------------------------------------------------------- +Pre-requisites: +- GStreamer-1.0 Development package +- GStreamer-1.0 Base Plugins Development package + +Install using: + sudo apt-get install libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev +-------------------------------------------------------------------------------- +Compiling and installing the plugin: +Run make and sudo make install + +NOTE: To compile the sources, run make with "sudo" or root permission. + +About: +This plugin performs batch meta mux for the same source and the "same" frame. +The "same" frame is based on the frame PTS. nvdsmetamux will try to find the +nearest frame PTS of the same source. There is PTS diff tolerance between the +PTS of the frame when search the nearest frame. Application can configure the +PTS diff tolerance. +Application can select the sink pad which video frame will be passed to src pad. +Application also can configure to select the source ids which output from one +model. The meta data will be merged if application select it. + + +Properties: + active-pad : Active sink pad which buffer will transfer to src pad + flags: readable, writable + String. Default: null + config-file : Preprocess Config File + flags: readable, writable + String. Default: null + latency : Additional latency in live mode to allow upstream to take longer to produce buffers for the current position (in nanoseconds) + flags: readable, writable + Unsigned Integer64. Range: 0 - 18446744073709551615 Default: 0 + min-upstream-latency: When sources with a higher latency are expected to be plugged in dynamically after the aggregator has started playing, this allows overriding the minimum latency reported by the initial source(s). This is only taken into account when larger than the actually reported minimum latency. (nanoseconds) + flags: readable, writable + Unsigned Integer64. Range: 0 - 18446744073709551615 Default: 0 + name : The name of the object + flags: readable, writable + String. Default: "nvdsmetamux0" + parent : The parent of the object + flags: readable, writable + Object of type "GstObject" + pts-tolerance : Time diff tolerance when search the same frame of the same source id in microseconds + flags: readable, writable + Integer64. Range: -9223372036854775808 - 9223372036854775807 Default: 60000 + start-time : Start time to use if start-time-selection=set + flags: readable, writable + Unsigned Integer64. Range: 0 - 18446744073709551615 Default: 18446744073709551615 + start-time-selection: Decides which start time is output + flags: readable, writable + Enum "GstAggregatorStartTimeSelection" Default: 0, "zero" + (0): zero - Start at 0 running time (default) + (1): first - Start at first observed input running time + (2): set - Set start time with start-time property + + +Configuration properties: + +Group config-key Description +------------------------------------------------------------------------------------------------------------------------------------------------------ +[propert] enable Enable the functions of MetaMux + active-pad Use the source from this pad to synchronize the sources from the branches + pts-tolerance When the difference between the branch source and the base source is larger than tolerance value, + meatamux will not combine the metadata into current output +[group] src-ids-model The source IDs list to be output for specified GIE. The GIE uique-id should be attached as the key + postfix. E.G. "src-ids-model-3: 0;1;3" means to output source 0, source 1 and source 3 inference result + from GIE with unique-id 3. + + +NOTE: +1. nvdsmetamux is alpha quality currently. +2. Please refer test/metamux.sh for more command which been tested. + +Run: +# mux batch meta which from different model. +gst-launch-1.0 nvstreammux name=m batch-size=4 width=1920 height=1080 ! queue ! nvdspreprocess config-file= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-preprocess-test/config_preprocess.txt ! nvinfer config-file-path= /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_primary.txt input-tensor-meta=1 batch-size=4 ! queue ! meta.sink_0 nvstreammux name=m2 batch-size=4 width=1920 height=1080 ! queue ! nvdspreprocess config-file= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-preprocess-test/config_preprocess_yoloV4.txt ! nvinfer config-file-path= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-preprocess-test/config_infer_primary_yoloV4.txt input-tensor-meta=1 batch-size=4 ! queue ! meta.sink_1 nvdsmetamux config-file=/opt/nvidia/deepstream/deepstream/sources/gst-plugins/gst-nvdsmetamux/config_metamux.txt name=meta ! nvmultistreamtiler width=1920 height=1080 ! nvvideoconvert ! nvdsosd ! queue ! nvvideoconvert ! queue ! x264enc ! queue ! rtph264pay config-interval=10 pt=96 ! udpsink host=10.19.225.205 port=5000 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_0 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_1 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_2 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_3 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m2.sink_0 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m2.sink_1 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m2.sink_2 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m2.sink_3 + +# mux batch meta which from different model with tee. +gst-launch-1.0 nvstreammux name=m batch-size=4 width=1920 height=1080 sync-inputs=0 batched-push-timeout= 100000 live-source=1 ! nvvideoconvert ! tee name=t t. ! nvvideoconvert ! video/x-raw(memory:NVMM),width=1920,height=1082 ! queue ! nvdspreprocess config-file= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-preprocess-test/config_preprocess.txt ! nvinfer config-file-path= /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_primary.txt input-tensor-meta=1 batch-size=4 ! queue ! meta.sink_0 t. ! queue ! nvdspreprocess config-file= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-preprocess-test/config_preprocess_yoloV4.txt ! nvinfer config-file-path= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-preprocess-test/config_infer_primary_yoloV4.txt input-tensor-meta=1 batch-size=4 ! queue ! meta.sink_1 nvdsmetamux config-file=/opt/nvidia/deepstream/deepstream/sources/gst-plugins/gst-nvdsmetamux/config_metamux.txt name=meta ! nvmultistreamtiler width=1920 height=1080 ! nvvideoconvert ! nvdsosd ! queue ! nvvideoconvert ! queue ! x264enc ! queue ! rtph264pay config-interval=10 pt=96 ! udpsink host=10.19.225.205 port=5000 uridecodebin3 uri=rtsp://10.19.225.227/media/video1 ! m.sink_0 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_1 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_2 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_3 + diff --git a/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo/Makefile b/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo/Makefile new file mode 100755 index 0000000..66dadd5 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo/Makefile @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CC:= g++ +NVCC:=/usr/local/cuda/bin/nvcc + +CFLAGS:= -Wall -std=c++11 -shared -fPIC -Wno-error=deprecated-declarations +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes -I/usr/local/cuda/include + +LIBS:= -lnvinfer_plugin -lnvinfer -L/usr/local/cuda/lib64 -lcudart -lcublas -lstdc++fs +LFLAGS:= -shared -Wl,--start-group $(LIBS) -Wl,--end-group + +INCS:= $(wildcard *.h) +SRCFILES:= nvdsparsebbox_Yolo.cpp + +TARGET_LIB:= libnvdsinfer_custom_impl_Yolo.so + +TARGET_OBJS:= $(SRCFILES:.cpp=.o) +TARGET_OBJS:= $(TARGET_OBJS:.cu=.o) + +all: $(TARGET_LIB) + +%.o: %.cpp $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +%.o: %.cu $(INCS) Makefile + $(NVCC) -c -o $@ --compiler-options '-fPIC' $< + +$(TARGET_LIB) : $(TARGET_OBJS) + $(CC) -o $@ $(TARGET_OBJS) $(LFLAGS) + +clean: + rm -rf $(TARGET_LIB) diff --git a/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo.cpp b/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo.cpp new file mode 100755 index 0000000..fa72b65 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo.cpp @@ -0,0 +1,138 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "nvdsinfer_custom_impl.h" + +static const int NUM_CLASSES_YOLO = 80; + +float clamp(const float val, const float minVal, const float maxVal) +{ + assert(minVal <= maxVal); + return std::min(maxVal, std::max(minVal, val)); +} + +extern "C" bool NvDsInferParseCustomYoloV4( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + NvDsInferParseDetectionParams const& detectionParams, + std::vector& objectList); + +extern "C" bool NvDsInferParseCustomYoloV4( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + NvDsInferParseDetectionParams const& detectionParams, + std::vector& objectList) +{ + if (NUM_CLASSES_YOLO != detectionParams.numClassesConfigured) + { + std::cerr << "WARNING: Num classes mismatch. Configured:" + << detectionParams.numClassesConfigured + << ", detected by network: " << NUM_CLASSES_YOLO << std::endl; + } + + const NvDsInferLayerInfo *boxes = nullptr; + const NvDsInferLayerInfo *scores = nullptr; + const NvDsInferLayerInfo *num = nullptr; + const NvDsInferLayerInfo *classes_layer = nullptr; + + for (size_t l = 0; l < outputLayersInfo.size(); l++) { + if (!strcmp(outputLayersInfo[l].layerName, "num_detections")) { + num = &outputLayersInfo[l]; + } + if (!strcmp(outputLayersInfo[l].layerName, "nmsed_boxes")) { + boxes = &outputLayersInfo[l]; + } + if (!strcmp(outputLayersInfo[l].layerName, "nmsed_scores")) { + scores = &outputLayersInfo[l]; + } + if (!strcmp(outputLayersInfo[l].layerName, "nmsed_classes")) { + classes_layer = &outputLayersInfo[l]; + } + } + + if (!boxes || !scores || !classes_layer) { + std::cerr << "ERROR: Missing required output layers (nmsed_boxes, nmsed_scores, nmsed_classes)" << std::endl; + return false; + } + + const float* bbox_buffer = (const float*)boxes->buffer; + const float* score_buffer = (const float*)scores->buffer; + const float* class_buffer = (const float*)classes_layer->buffer; + + // Get number of detections + uint num_bboxes; + if (num) { + num_bboxes = ((int*)num->buffer)[0]; + } else { + num_bboxes = boxes->inferDims.d[0]; + } + + for (uint n = 0; n < num_bboxes; ++n) { + int class_id = static_cast(class_buffer[n]); + + // Validate class_id + if (class_id < 0 || class_id >= (int)detectionParams.numClassesConfigured) { + continue; + } + + float score = score_buffer[n]; + if (score < detectionParams.perClassPreclusterThreshold[class_id]) { + continue; + } + + // Parse bbox: [x1, y1, x2, y2] normalized coordinates + float bx1 = bbox_buffer[n * 4]; + float by1 = bbox_buffer[n * 4 + 1]; + float bx2 = bbox_buffer[n * 4 + 2]; + float by2 = bbox_buffer[n * 4 + 3]; + + // Convert to pixel coordinates + float x1 = clamp(bx1 * networkInfo.width, 0.0f, (float)networkInfo.width); + float y1 = clamp(by1 * networkInfo.height, 0.0f, (float)networkInfo.height); + float x2 = clamp(bx2 * networkInfo.width, 0.0f, (float)networkInfo.width); + float y2 = clamp(by2 * networkInfo.height, 0.0f, (float)networkInfo.height); + + NvDsInferParseObjectInfo outObj; + outObj.left = x1; + outObj.top = y1; + outObj.width = clamp(x2 - x1, 0.0f, (float)networkInfo.width); + outObj.height = clamp(y2 - y1, 0.0f, (float)networkInfo.height); + outObj.classId = class_id; + outObj.rotation_angle = 0.0f; + outObj.detectionConfidence = score; + + if (outObj.width < 1 || outObj.height < 1) { + continue; + } + + objectList.push_back(outObj); + } + + return true; +} +/* YOLOv4 implementations end*/ + + +/* Check that the custom function has been defined correctly */ +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomYoloV4); diff --git a/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser/Makefile b/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser/Makefile new file mode 100755 index 0000000..24204b7 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser/Makefile @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +CC:= g++ + +CFLAGS:= -Wall -Werror -std=c++11 -shared -fPIC -Wno-error=deprecated-declarations + +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes + +LIBS:= -lnvinfer +LFLAGS:= -Wl,--start-group $(LIBS) -Wl,--end-group + +SRCFILES:= nvinfer_custom_lpr_parser.cpp +TARGET_LIB:= libnvdsinfer_custom_impl_lpr.so + +all: $(TARGET_LIB) + +$(TARGET_LIB) : $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS) $(LFLAGS) + +clean: + rm -rf $(TARGET_LIB) diff --git a/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser/nvinfer_custom_lpr_parser.cpp b/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser/nvinfer_custom_lpr_parser.cpp new file mode 100755 index 0000000..dca1f52 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser/nvinfer_custom_lpr_parser.cpp @@ -0,0 +1,142 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "nvdsinfer.h" +#include + +using namespace std; +using std::string; +using std::vector; + +static bool dict_ready=false; +std::vector dict_table; + +extern "C" +{ + +bool NvDsInferParseCustomNVPlate(std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, float classifierThreshold, + std::vector &attrList, std::string &attrString) +{ + int *outputStrBuffer = NULL; + float *outputConfBuffer = NULL; + NvDsInferAttribute LPR_attr; + + int seq_len = 0; + + // Get list + vector str_idxes; + int prev = 100; + + // For confidence + double bank_softmax_max[16] = {0.0}; + unsigned int valid_bank_count = 0; + bool do_softmax = false; + ifstream fdict; + + setlocale(LC_CTYPE, ""); + + if(!dict_ready) { + fdict.open("dict.txt"); + if(!fdict.is_open()) + { + cout << "open dictionary file failed." << endl; + return false; + } + while(!fdict.eof()) { + string strLineAnsi; + if ( getline(fdict, strLineAnsi) ) { + dict_table.push_back(strLineAnsi); + } + } + dict_ready=true; + fdict.close(); + } + + int layer_size = outputLayersInfo.size(); + + LPR_attr.attributeConfidence = 1.0; + + seq_len = networkInfo.width/4; + + for( int li=0; li(outputLayersInfo[li].buffer); + } + else if (outputLayersInfo[li].dataType == 3) { + if(!outputStrBuffer) + outputStrBuffer = static_cast(outputLayersInfo[li].buffer); + } + } + } + + for(int seq_id = 0; seq_id < seq_len; seq_id++) { + do_softmax = false; + + int curr_data = outputStrBuffer[seq_id]; + if (seq_id == 0) { + prev = curr_data; + str_idxes.push_back(curr_data); + if ( curr_data != static_cast(dict_table.size()) ) do_softmax = true; + } else { + if (curr_data != prev) { + str_idxes.push_back(curr_data); + if (static_cast(curr_data) != dict_table.size()) do_softmax = true; + } + prev = curr_data; + } + + // Do softmax + if (do_softmax) { + do_softmax = false; + bank_softmax_max[valid_bank_count] = outputConfBuffer[curr_data]; + valid_bank_count++; + } + } + + attrString = ""; + for(unsigned int id = 0; id < str_idxes.size(); id++) { + if (static_cast(str_idxes[id]) != dict_table.size()) { + attrString += dict_table[str_idxes[id]]; + } + } + + //Ignore the short string, it may be wrong plate string + if (valid_bank_count >= 3) { + + LPR_attr.attributeIndex = 0; + LPR_attr.attributeValue = 1; + LPR_attr.attributeLabel = strdup(attrString.c_str()); + for (unsigned int count = 0; count < valid_bank_count; count++) { + LPR_attr.attributeConfidence *= bank_softmax_max[count]; + } + attrList.push_back(LPR_attr); + } + + return true; +} + +}//end of extern "C" diff --git a/deepstream_parallel_inference_app/tritonserver/build_engine.sh b/deepstream_parallel_inference_app/tritonserver/build_engine.sh new file mode 100755 index 0000000..3f5bda6 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonserver/build_engine.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +IS_JETSON_PLATFORM=`uname -i | grep aarch64` + +export PATH=$PATH:/usr/src/tensorrt/bin + +trtexec --fp16 --onnx=./models/yolov4/1/yolov4_-1_3_416_416_dynamic.onnx.nms.onnx --saveEngine=./models/yolov4/1/yolov4_-1_3_416_416_dynamic.onnx_b32_gpu0.engine --minShapes=input:1x3x416x416 --optShapes=input:16x3x416x416 --maxShapes=input:32x3x416x416 --shapes=input:16x3x416x416 + +mkdir -p models/trafficcamnet/1/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/trafficcamnet/pruned_onnx_v1.0.4/files?redirect=true&path=resnet18_trafficcamnet_pruned.onnx' -O ./models/trafficcamnet/1/resnet18_trafficcamnet_pruned.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/trafficcamnet/pruned_onnx_v1.0.4/files?redirect=true&path=labels.txt' -O ./models/trafficcamnet/labels.txt +trtexec --onnx=./models/trafficcamnet/1/resnet18_trafficcamnet_pruned.onnx --fp16 \ + --saveEngine=./models/trafficcamnet/1/resnet18_trafficcamnet_pruned.onnx_b8_gpu0_fp16.engine --minShapes="input_1:0":1x3x544x960 \ + --optShapes="input_1:0":4x3x544x960 --maxShapes="input_1:0":8x3x544x960 + +mkdir -p models/US_LPD/1/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/lpdnet/pruned_v2.3.1/files?redirect=true&path=LPDNet_usa_pruned_tao5.onnx' -O models/US_LPD/1/LPDNet_usa_pruned_tao5.onnx +wget 'https://api.ngc.nvidia.com/v2/models/nvidia/tao/lpdnet/versions/pruned_v1.0/files/usa_lpd_label.txt' -O models/US_LPD/usa_lpd_label.txt +trtexec --onnx=models/US_LPD/1/LPDNet_usa_pruned_tao5.onnx --fp16 \ + --saveEngine=models/US_LPD/1//LPDNet_usa_pruned_tao5.onnx_b16_gpu0_fp16.engine --minShapes="input_1:0":1x3x480x640 \ + --optShapes="input_1:0":16x3x480x640 --maxShapes="input_1:0":16x3x480x640 + +mkdir models/us_lprnet/1/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/lprnet/deployable_onnx_v1.1/files?redirect=true&path=us_lprnet_baseline18_deployable.onnx' -O models/us_lprnet/1/us_lprnet_baseline18_deployable.onnx +trtexec --onnx=models/us_lprnet/1/us_lprnet_baseline18_deployable.onnx --fp16 \ + --saveEngine=models/us_lprnet/1/us_lprnet_baseline18_deployable.onnx_b16_gpu0_fp16.engine --minShapes="image_input":1x3x48x96 \ + --optShapes="image_input":8x3x48x96 --maxShapes="image_input":16x3x48x96 + +mkdir -p models/peoplenet/1/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/deployable_quantized_onnx_v2.6.3/files?redirect=true&path=resnet34_peoplenet.onnx' \ + -O models/peoplenet/1/resnet34_peoplenet.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/deployable_quantized_onnx_v2.6.3/files?redirect=true&path=labels.txt' \ + -O models/peoplenet/labels.txt +trtexec --onnx=./models/peoplenet/1/resnet34_peoplenet.onnx --fp16 \ + --saveEngine=./models/peoplenet/1/resnet34_peoplenet.onnx_b8_gpu0_fp16.engine \ + --minShapes="input_1:0":1x3x544x960 --optShapes="input_1:0":8x3x544x960 --maxShapes="input_1:0":8x3x544x960 + +#generate engine for vehicle related models. +echo "Building Model Secondary_CarMake..." +mkdir -p models/Secondary_CarMake/1/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehiclemakenet/pruned_onnx_v1.1.0/files?redirect=true&path=resnet18_pruned.onnx' \ + -O models/Secondary_CarMake/1/resnet18_pruned.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehiclemakenet/pruned_onnx_v1.1.0/files?redirect=true&path=labels.txt' \ + -O models/Secondary_CarMake/labels.txt +trtexec --onnx=models/Secondary_CarMake/1/resnet18_pruned.onnx --fp16 \ + --saveEngine=models/Secondary_CarMake/1/resnet18_pruned.onnx_b16_gpu0_fp16.engine --minShapes="input_1:0":1x3x224x224 \ + --optShapes="input_1:0":8x3x224x224 --maxShapes="input_1:0":16x3x224x224 + +echo "Building Model Secondary_VehicleTypes..." +mkdir -p models/Secondary_VehicleTypes/1/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehicletypenet/pruned_onnx_v1.1.0/files?redirect=true&path=resnet18_pruned.onnx' \ + -O models/Secondary_VehicleTypes/1/resnet18_pruned.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehicletypenet/pruned_onnx_v1.1.0/files?redirect=true&path=labels.txt' \ + -O models/Secondary_VehicleTypes/labels.txt +trtexec --onnx=models/Secondary_VehicleTypes/1/resnet18_pruned.onnx --fp16 \ + --saveEngine=models/Secondary_VehicleTypes/1/resnet18_pruned.onnx_b16_gpu0_fp16.engine --minShapes="input_1:0":1x3x224x224 \ + --optShapes="input_1:0":8x3x224x224 --maxShapes="input_1:0":16x3x224x224 +echo "Finished generating engine files." diff --git a/deepstream_parallel_inference_app/tritonserver/models/Secondary_CarMake/config.pbtxt b/deepstream_parallel_inference_app/tritonserver/models/Secondary_CarMake/config.pbtxt new file mode 100755 index 0000000..c2d6d0a --- /dev/null +++ b/deepstream_parallel_inference_app/tritonserver/models/Secondary_CarMake/config.pbtxt @@ -0,0 +1,43 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: "Secondary_CarMake" +platform: "tensorrt_plan" +max_batch_size: 16 +default_model_filename: "resnet18_pruned.onnx_b16_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [3, 224, 224] + } +] +output [ + { + name: "predictions/Softmax:0" + data_type: TYPE_FP32 + dims: [ 20 ] + } +] +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/deepstream_parallel_inference_app/tritonserver/models/Secondary_VehicleTypes/config.pbtxt b/deepstream_parallel_inference_app/tritonserver/models/Secondary_VehicleTypes/config.pbtxt new file mode 100755 index 0000000..011735a --- /dev/null +++ b/deepstream_parallel_inference_app/tritonserver/models/Secondary_VehicleTypes/config.pbtxt @@ -0,0 +1,43 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: "Secondary_VehicleTypes" +platform: "tensorrt_plan" +max_batch_size: 16 +default_model_filename: "resnet18_pruned.onnx_b16_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [3, 224, 224] + } +] +output [ + { + name: "predictions/Softmax:0" + data_type: TYPE_FP32 + dims: [6] + } +] +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/deepstream_parallel_inference_app/tritonserver/models/US_LPD/config.pbtxt b/deepstream_parallel_inference_app/tritonserver/models/US_LPD/config.pbtxt new file mode 100755 index 0000000..9a7ff7e --- /dev/null +++ b/deepstream_parallel_inference_app/tritonserver/models/US_LPD/config.pbtxt @@ -0,0 +1,50 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: "US_LPD" +platform: "tensorrt_plan" +max_batch_size: 16 +default_model_filename: "LPDNet_usa_pruned_tao5.onnx_b16_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [ 3, 480, 640] + } +] +output [ + { + name: "output_bbox/BiasAdd:0" + data_type: TYPE_FP32 + dims: [4, 30, 40] + }, + + { + name: "output_cov/Sigmoid:0" + data_type: TYPE_FP32 + dims: [1, 30, 40] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/deepstream_parallel_inference_app/tritonserver/models/US_LPD/usa_lpd_label.txt b/deepstream_parallel_inference_app/tritonserver/models/US_LPD/usa_lpd_label.txt new file mode 100755 index 0000000..95ffacc --- /dev/null +++ b/deepstream_parallel_inference_app/tritonserver/models/US_LPD/usa_lpd_label.txt @@ -0,0 +1 @@ +lpd diff --git a/deepstream_parallel_inference_app/tritonserver/models/bodypose2d/1/model.onnx b/deepstream_parallel_inference_app/tritonserver/models/bodypose2d/1/model.onnx new file mode 100755 index 0000000..9ee732c --- /dev/null +++ b/deepstream_parallel_inference_app/tritonserver/models/bodypose2d/1/model.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57f0a14a6c6dcf619c8702b5cb4368730626f7d824b603bf0ba595d1cd78627f +size 83063809 diff --git a/deepstream_parallel_inference_app/tritonserver/models/bodypose2d/config.pbtxt b/deepstream_parallel_inference_app/tritonserver/models/bodypose2d/config.pbtxt new file mode 100755 index 0000000..901d52a --- /dev/null +++ b/deepstream_parallel_inference_app/tritonserver/models/bodypose2d/config.pbtxt @@ -0,0 +1,37 @@ +name: "bodypose2d" +platform: "onnxruntime_onnx" +backend: "onnxruntime" +max_batch_size: 4 +input [ + { + name: "input" + data_type: TYPE_FP32 + dims: [ + 3, + 224, + 224 + ] + } +] +output [ + { + name: "266" + data_type: TYPE_FP32 + dims: [ + 1 + ] + }, + { + name: "268", + data_type: TYPE_FP32 + dims: [ + 1 + ] + } +] + +instance_group { + count: 1 + gpus: 0 + kind: KIND_GPU +} diff --git a/deepstream_parallel_inference_app/tritonserver/models/peoplenet/config.pbtxt b/deepstream_parallel_inference_app/tritonserver/models/peoplenet/config.pbtxt new file mode 100755 index 0000000..75f713c --- /dev/null +++ b/deepstream_parallel_inference_app/tritonserver/models/peoplenet/config.pbtxt @@ -0,0 +1,48 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: "peoplenet" +platform: "tensorrt_plan" +max_batch_size: 8 +default_model_filename: "resnet34_peoplenet.onnx_b8_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + dims: [ 3, 544, 960 ] + } +] +output [ + { + name: "output_bbox/BiasAdd:0" + data_type: TYPE_FP32 + dims: [ 12, 34, 60 ] + }, + { + name: "output_cov/Sigmoid:0" + data_type: TYPE_FP32 + dims: [ 3, 34, 60 ] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/deepstream_parallel_inference_app/tritonserver/models/peoplenet/labels.txt b/deepstream_parallel_inference_app/tritonserver/models/peoplenet/labels.txt new file mode 100755 index 0000000..e69de29 diff --git a/deepstream_parallel_inference_app/tritonserver/models/trafficcamnet/config.pbtxt b/deepstream_parallel_inference_app/tritonserver/models/trafficcamnet/config.pbtxt new file mode 100755 index 0000000..dfaa023 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonserver/models/trafficcamnet/config.pbtxt @@ -0,0 +1,49 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: "trafficcamnet" +platform: "tensorrt_plan" +max_batch_size: 8 +default_model_filename: "resnet18_trafficcamnet_pruned.onnx_b8_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [3, 544, 960] + } +] +output [ + { + name: "output_bbox/BiasAdd:0" + data_type: TYPE_FP32 + dims: [16, 34, 60] + }, + + { + name: "output_cov/Sigmoid:0" + data_type: TYPE_FP32 + dims: [4, 34, 60] + } +] +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/deepstream_parallel_inference_app/tritonserver/models/trafficcamnet/labels.txt b/deepstream_parallel_inference_app/tritonserver/models/trafficcamnet/labels.txt new file mode 100755 index 0000000..1a20095 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonserver/models/trafficcamnet/labels.txt @@ -0,0 +1,4 @@ +car +bicycle +person +road_sign diff --git a/deepstream_parallel_inference_app/tritonserver/models/us_lprnet/config.pbtxt b/deepstream_parallel_inference_app/tritonserver/models/us_lprnet/config.pbtxt new file mode 100755 index 0000000..548ee9a --- /dev/null +++ b/deepstream_parallel_inference_app/tritonserver/models/us_lprnet/config.pbtxt @@ -0,0 +1,50 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: "us_lprnet" +platform: "tensorrt_plan" +max_batch_size: 16 +default_model_filename: "us_lprnet_baseline18_deployable.onnx_b16_gpu0_fp16.engine" +input [ + { + name: "image_input" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [ 3, 48, 96] + } +] +output [ + { + name: "tf_op_layer_ArgMax" + data_type: TYPE_INT32 + dims: [24] + }, + + { + name: "tf_op_layer_Max" + data_type: TYPE_FP32 + dims: [ 24 ] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/deepstream_parallel_inference_app/tritonserver/models/yolov4/1/yolov4_-1_3_416_416_dynamic.onnx.nms.onnx b/deepstream_parallel_inference_app/tritonserver/models/yolov4/1/yolov4_-1_3_416_416_dynamic.onnx.nms.onnx new file mode 100755 index 0000000..f1eed36 --- /dev/null +++ b/deepstream_parallel_inference_app/tritonserver/models/yolov4/1/yolov4_-1_3_416_416_dynamic.onnx.nms.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b63d119949293621a3b7a901f0cbfc07580ef11569dbe7b4aaa6a986ad0b7230 +size 257499192 diff --git a/legacy_apps/.gitkeep b/legacy_apps/.gitkeep new file mode 100755 index 0000000..e69de29 diff --git a/legacy_apps/back-to-back-detectors/.backtobackdetectors.png b/legacy_apps/back-to-back-detectors/.backtobackdetectors.png new file mode 100755 index 0000000..baa587e Binary files /dev/null and b/legacy_apps/back-to-back-detectors/.backtobackdetectors.png differ diff --git a/legacy_apps/back-to-back-detectors/.backtobackdetectors_pipeline.png b/legacy_apps/back-to-back-detectors/.backtobackdetectors_pipeline.png new file mode 100755 index 0000000..0761ecc Binary files /dev/null and b/legacy_apps/back-to-back-detectors/.backtobackdetectors_pipeline.png differ diff --git a/legacy_apps/back-to-back-detectors/Makefile b/legacy_apps/back-to-back-detectors/Makefile new file mode 100755 index 0000000..93ec906 --- /dev/null +++ b/legacy_apps/back-to-back-detectors/Makefile @@ -0,0 +1,61 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +APP:= back-to-back-detectors + +TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -) + +DS_SDK_ROOT:=/opt/nvidia/deepstream/deepstream + +LIB_INSTALL_DIR?=$(DS_SDK_ROOT)/lib/ + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= -I$(DS_SDK_ROOT)/sources/includes \ + -I /usr/local/cuda-$(CUDA_VER)/include + +CFLAGS+= `pkg-config --cflags $(PKGS)` + +LIBS:= `pkg-config --libs $(PKGS)` + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta \ + -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart \ + -lcuda -Wl,-rpath,$(LIB_INSTALL_DIR) + +all: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +clean: + rm -rf $(OBJS) $(APP) + + diff --git a/legacy_apps/back-to-back-detectors/README.md b/legacy_apps/back-to-back-detectors/README.md new file mode 100755 index 0000000..583056a --- /dev/null +++ b/legacy_apps/back-to-back-detectors/README.md @@ -0,0 +1,89 @@ +# BACK-TO-BACK-DETECTORS REFERENCE APP USING DEEPSTREAMSDK 9.0 + +## Introduction +The project contains Back to Back detector application to show the +capability of Deepstream SDK. + +This sample builds on top of the deepstream-test1 sample to demonstrate how to +add multiple back-to-back detectors in the pipeline. + +Two instances of "nvinfer" or "nvinferserver" element are added to the pipeline serially after +nvstreammux and before the display components. Both the "nvinfer" or "nvinferserver" instances have +their own config files. + +The first "nvinfer" or "nvinferserver" instance (Person/Vehicle/Bicycle/RoadSign) will always act +as primary detector. + +The second "nvinfer" or "nvinferserver" instance (Face Detection) can be configured as +primary(full-frame) / secondary (operating on primary detected objects). By +default it is configured in the secondary mode. To change the second "nvinfer" or "nvinferserver" +instance to primary mode, change the macro `SECOND_DETECTOR_IS_SECONDARY` in the +sources to 0. + +## Prequisites: + +Please follow instructions in the `apps/sample_apps/deepstream-app/README` on how +to install the prequisites for Deepstream SDK, the DeepStream SDK itself and the +apps. + +## Getting Started + +- Preferably clone the app in + `/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/` + +- Edit the `primary_detector_config.txt` according to the location of the models to be used + +## Steps to download the models: +- To download the models for the second nvinfer, visit: + https://github.com/NVIDIA-AI-IOT/redaction_with_deepstream +- Use the following commands: +``` + $ cd /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/ + $ cd deepstream_reference_apps/deepstream-app-tao-configs/ + $ sudo cp -a * /opt/nvidia/deepstream/deepstream/samples/configs/tao_pretrained_models/ + $ sudo apt install -y wget zip + $ cd /opt/nvidia/deepstream/deepstream/samples/configs/tao_pretrained_models/ + $ sudo ./download_models.sh + +- Setup Triton model repository: + $ cd /opt/nvidia/deepstream/deepstream/samples/ +- Run prepare_ds_triton_model_repo.sh script to create Primary infer model "PrimaryDetector" + $ ./prepare_ds_triton_model_repo.sh +- Run prepare_ds_triton_tao_model_repo.sh script to create Secondary infer model "FaceNet" + $ ./prepare_ds_triton_tao_model_repo.sh + +``` + +Back to back detectors app pipeline: +![DS Back to back detectors Pipeline](.backtobackdetectors_pipeline.png) + +The result should be like below: +![DS Back to back detectors Screenshot](.backtobackdetectors.png) +## Compilation Steps and Execution: +``` + $ Set CUDA_VER in the MakeFile as per platform. + For x86, CUDA_VER=13.1 + For Jetson, CUDA_VER=13.0 + $ sudo make + + $ ./back-to-back-detectors + Ex.: ./back-to-back-detectors /opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.h264 + +Use option "-t inferserver" to select nvinferserver as the inference plugin + $ ./back-to-back-detectors -t inferserver + Ex.: ./back-to-back-detectors -t inferserver /opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.h264 + +NOTE: +- For Jetson, first run the below commands + $ cd /opt/nvidia/deepstream/deepstream/samples/triton_tao_model_repo + $ sudo ln -s ../triton_model_repo/Primary_Detector . +Then run the app. + +``` + +NOTE: +- Run the above commands with sudo. +- Edit the paths in `secondary_detector_config.txt` to the location of the models + downloaded from the above site. +- back-to-back-detectors application does not run inside jetson triton docker. + diff --git a/legacy_apps/back-to-back-detectors/back_to_back_detectors.c b/legacy_apps/back-to-back-detectors/back_to_back_detectors.c new file mode 100755 index 0000000..f4de152 --- /dev/null +++ b/legacy_apps/back-to-back-detectors/back_to_back_detectors.c @@ -0,0 +1,383 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "gstnvdsmeta.h" +#include + +#define MAX_DISPLAY_LEN 64 + +#define PGIE_CLASS_ID_VEHICLE 0 +#define PGIE_CLASS_ID_PERSON 2 + +#define SGIE_CLASS_ID_LP 1 +#define SGIE_CLASS_ID_FACE 0 + +/* Change this to 0 to make the 2nd detector act as a primary(full-frame) detector. + * When set to 1, it will act as secondary(operates on primary detected objects). */ +#define SECOND_DETECTOR_IS_SECONDARY 1 + +#define NVINFER_PLUGIN "nvinfer" +#define NVINFERSERVER_PLUGIN "nvinferserver" + +#define INFER_PGIE_CONFIG_FILE "primary_detector_config.txt" +#define INFER_SGIE_CONFIG_FILE "config_infer_primary_yoloV8_face.txt" +#define INFERSERVER_PGIE_CONFIG_FILE "inferserver/primary_detector_config.txt" +#define INFERSERVER_SGIE_CONFIG_FILE "inferserver/secondary_detector_config.txt" + +/* The muxer output resolution must be set if the input streams will be of + * different resolution. The muxer will scale all the input frames to this + * resolution. */ +#define MUXER_OUTPUT_WIDTH 1280 +#define MUXER_OUTPUT_HEIGHT 720 + +/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set + * based on the fastest source's framerate. */ +#define MUXER_BATCH_TIMEOUT_USEC 40000 + +gint frame_number = 0; +gchar pgie_classes_str[4][32] = { "Vehicle", "TwoWheeler", "Person", + "Roadsign" +}; + +#define PRIMARY_DETECTOR_UID 1 +#define SECONDARY_DETECTOR_UID 2 + +/* nvvidconv_sink_pad_buffer_probe will extract metadata received on nvvideoconvert sink pad + * and update params for drawing rectangle, object information etc. */ + +static GstPadProbeReturn +nvvidconv_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, + gpointer u_data) +{ + GstBuffer *buf = (GstBuffer *) info->data; + NvDsObjectMeta *obj_meta = NULL; + guint vehicle_count = 0; + guint person_count = 0; + guint face_count = 0; + NvDsMetaList * l_frame = NULL; + NvDsMetaList * l_obj = NULL; + NvDsDisplayMeta *display_meta = NULL; + + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf); + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data); + int offset = 0; + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) { + obj_meta = (NvDsObjectMeta *) (l_obj->data); + + /* Check that the object has been detected by the primary detector + * and that the class id is that of vehicles/persons. */ + if (obj_meta->unique_component_id == PRIMARY_DETECTOR_UID) { + if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) + vehicle_count++; + if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) + person_count++; + } + + if (obj_meta->unique_component_id == SECONDARY_DETECTOR_UID) { + if (obj_meta->class_id == SGIE_CLASS_ID_FACE) { + face_count++; + /* Print this info only when operating in secondary model. */ + if (SECOND_DETECTOR_IS_SECONDARY) + g_print ("Face found for parent object %p (type=%s)\n", + obj_meta->parent, pgie_classes_str[obj_meta->parent->class_id]); + } + } + } + display_meta = nvds_acquire_display_meta_from_pool(batch_meta); + NvOSD_TextParams *txt_params = &display_meta->text_params[0]; + display_meta->num_labels = 1; + txt_params->display_text = g_malloc0 (MAX_DISPLAY_LEN); + offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "Person = %d ", person_count); + offset += snprintf(txt_params->display_text + offset , MAX_DISPLAY_LEN, "Vehicle = %d ", vehicle_count); + offset += snprintf(txt_params->display_text + offset , MAX_DISPLAY_LEN, "Face = %d ", face_count); + + /* Now set the offsets where the string should appear */ + txt_params->x_offset = 10; + txt_params->y_offset = 12; + + /* Font , font-color and font-size */ + txt_params->font_params.font_name = "Serif"; + txt_params->font_params.font_size = 10; + txt_params->font_params.font_color.red = 1.0; + txt_params->font_params.font_color.green = 1.0; + txt_params->font_params.font_color.blue = 1.0; + txt_params->font_params.font_color.alpha = 1.0; + + /* Text background color */ + txt_params->set_bg_clr = 1; + txt_params->text_bg_clr.red = 0.0; + txt_params->text_bg_clr.green = 0.0; + txt_params->text_bg_clr.blue = 0.0; + txt_params->text_bg_clr.alpha = 1.0; + + nvds_add_display_meta_to_frame(frame_meta, display_meta); + } + + + g_print ("Frame Number = %d Vehicle Count = %d Person Count = %d" + " Face Count = %d\n", + frame_number, vehicle_count, person_count, + face_count); + frame_number++; + return GST_PAD_PROBE_OK; +} + +static gboolean +bus_call (GstBus * bus, GstMessage * msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + g_print ("End of stream\n"); + g_main_loop_quit (loop); + break; + case GST_MESSAGE_ERROR:{ + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + default: + break; + } + return TRUE; +} + +static void +usage(const char *bin) +{ + g_printerr ("Usage: %s \n", bin); + g_printerr ("For nvinferserver, Usage: %s -t inferserver \n", bin); +} + +int +main (int argc, char *argv[]) +{ + GMainLoop *loop = NULL; + GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, + *decoder = NULL, *streammux = NULL, *sink = NULL, *primary_detector = NULL, + *secondary_detector = NULL, *nvvidconv = NULL, *nvosd = NULL; + GstBus *bus = NULL; + guint bus_watch_id; + GstPad *nvvidconv_sink_pad = NULL; + gboolean is_nvinfer_server = FALSE; + gchar *input_stream = NULL; + const char *infer_plugin = NVINFER_PLUGIN; + + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + /* Check input arguments */ + if (argc < 2) { + usage(argv[0]); + return -1; + } + + if (argc >=2 && !strcmp("-t", argv[1])) { + if (!strcmp("inferserver", argv[2])) { + is_nvinfer_server = TRUE; + } else { + usage(argv[0]); + return -1; + } + g_print ("Using nvinferserver as the inference plugin\n"); + } + + if (is_nvinfer_server) { + infer_plugin = NVINFERSERVER_PLUGIN; + } + + /* Standard GStreamer initialization */ + gst_init (&argc, &argv); + loop = g_main_loop_new (NULL, FALSE); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + pipeline = gst_pipeline_new ("pipeline"); + + /* Source element for reading from the file */ + source = gst_element_factory_make ("filesrc", "file-source"); + + /* Since the data format in the input file is elementary h264 stream, + * we need a h264parser */ + h264parser = gst_element_factory_make ("h264parse", "h264-parser"); + + /* Use nvdec_h264 for hardware accelerated decode on GPU */ + decoder = gst_element_factory_make ("nvv4l2decoder", "nvv4l2-decoder"); + + /* Create nvstreammux instance to form batches from one or more sources. */ + streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); + + if (!pipeline || !streammux) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + + /* Create two nvinfer instances for the two back-to-back detectors */ + primary_detector = gst_element_factory_make (infer_plugin, "primary-nvinference-engine1"); + + secondary_detector = gst_element_factory_make (infer_plugin, "primary-nvinference-engine2"); + + /* Use convertor to convert from NV12 to RGBA as required by nvosd */ + nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); + + /* Create OSD to draw on the converted RGBA buffer */ + nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); + + /* Finally render the osd output */ + if(prop.integrated) { + sink = gst_element_factory_make ("nv3dsink", "nvvideo-renderer"); + } else { +#ifdef __aarch64__ + sink = gst_element_factory_make ("nv3dsink", "nvvideo-renderer"); +#else + sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer"); +#endif + } + + if (!source || !h264parser || !decoder || !primary_detector || !secondary_detector + || !nvvidconv || !nvosd || !sink) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + + /* we set the input filename to the source element */ + if (is_nvinfer_server) { + input_stream = argv[3]; + g_object_set (G_OBJECT (source), "location", argv[3], NULL); + } else { + input_stream = argv[1]; + g_object_set (G_OBJECT (source), "location", argv[1], NULL); + } + + g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", + MUXER_OUTPUT_HEIGHT, "batch-size", 1, + "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); + + /* Set the config files for the two detectors. We demonstrate this by using + * the same detector model twice but making them act as vehicle-only and + * person-only detectors by adjusting the bbox confidence thresholds in the + * two seperate config files. */ + if (is_nvinfer_server) { + g_object_set (G_OBJECT (primary_detector), "config-file-path", INFERSERVER_PGIE_CONFIG_FILE, + "unique-id", PRIMARY_DETECTOR_UID, NULL); + } else { + g_object_set (G_OBJECT (primary_detector), "config-file-path", INFER_PGIE_CONFIG_FILE, + "unique-id", PRIMARY_DETECTOR_UID, NULL); + } + + if (is_nvinfer_server) { + g_object_set (G_OBJECT (secondary_detector), "config-file-path", INFERSERVER_SGIE_CONFIG_FILE, + "unique-id", SECONDARY_DETECTOR_UID, "process-mode", SECOND_DETECTOR_IS_SECONDARY ? 2 : 1, NULL); + } else { + g_object_set (G_OBJECT (secondary_detector), "config-file-path", INFER_SGIE_CONFIG_FILE, + "unique-id", SECONDARY_DETECTOR_UID, "process-mode", SECOND_DETECTOR_IS_SECONDARY ? 2 : 1, NULL); + } + + /* we add a message handler */ + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); + gst_object_unref (bus); + + /* Set up the pipeline */ + /* we add all elements into the pipeline */ + gst_bin_add_many (GST_BIN (pipeline), + source, h264parser, decoder, streammux, primary_detector, secondary_detector, + nvvidconv, nvosd, sink, NULL); + + GstPad *sinkpad, *srcpad; + gchar pad_name_sink[16] = "sink_0"; + gchar pad_name_src[16] = "src"; + + sinkpad = gst_element_get_request_pad (streammux, pad_name_sink); + if (!sinkpad) { + g_printerr ("Streammux request sink pad failed. Exiting.\n"); + return -1; + } + + srcpad = gst_element_get_static_pad (decoder, pad_name_src); + if (!srcpad) { + g_printerr ("Decoder request src pad failed. Exiting.\n"); + return -1; + } + + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Failed to link decoder to stream muxer. Exiting.\n"); + return -1; + } + + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + /* we link the elements together */ + /* file-source -> h264-parser -> nvh264-decoder -> + * pgie -> nvvidconv -> nvosd -> video-renderer */ + + if (!gst_element_link_many (source, h264parser, decoder, NULL)) { + g_printerr ("Elements could not be linked: 1. Exiting.\n"); + return -1; + } + + if (!gst_element_link_many (streammux, primary_detector, secondary_detector, + nvvidconv, nvosd, sink, NULL)) { + g_printerr ("Elements could not be linked: 2. Exiting.\n"); + return -1; + } + + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the nvvideoconvert element, since by that time, the buffer would have + * had got all the metadata. */ + nvvidconv_sink_pad = gst_element_get_static_pad (nvvidconv, "sink"); + if (!nvvidconv_sink_pad) + g_print ("Unable to get sink pad\n"); + else + gst_pad_add_probe (nvvidconv_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + nvvidconv_sink_pad_buffer_probe, NULL, NULL); + + /* Set the pipeline to "playing" state */ + g_print ("Now playing: %s\n", input_stream); + gst_element_set_state (pipeline, GST_STATE_PLAYING); + + /* Wait till pipeline encounters an error or EOS */ + g_print ("Running...\n"); + g_main_loop_run (loop); + + /* Out of the main loop, clean up nicely */ + g_print ("Returned, stopping playback\n"); + gst_element_set_state (pipeline, GST_STATE_NULL); + g_print ("Deleting pipeline\n"); + gst_object_unref (GST_OBJECT (pipeline)); + g_source_remove (bus_watch_id); + g_main_loop_unref (loop); + return 0; +} diff --git a/legacy_apps/back-to-back-detectors/config_infer_primary_yoloV8_face.txt b/legacy_apps/back-to-back-detectors/config_infer_primary_yoloV8_face.txt new file mode 100644 index 0000000..a8db310 --- /dev/null +++ b/legacy_apps/back-to-back-detectors/config_infer_primary_yoloV8_face.txt @@ -0,0 +1,26 @@ +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +model-color-format=0 +onnx-file=yolov8n-face.onnx +model-engine-file=yolov8n-face.onnx_b1_gpu0_fp32.engine +#int8-calib-file=calib.table +labelfile-path=labels.txt +batch-size=1 +network-mode=0 +num-detected-classes=1 +interval=0 +gie-unique-id=2 +process-mode=1 +network-type=3 +cluster-mode=4 +maintain-aspect-ratio=1 +symmetric-padding=1 +#workspace-size=2000 +parse-bbox-instance-mask-func-name=NvDsInferParseYoloFace +custom-lib-path=nvdsinfer_custom_impl_Yolo_face/libnvdsinfer_custom_impl_Yolo_face.so +output-instance-mask=1 + +[class-attrs-all] +pre-cluster-threshold=0.25 +topk=300 diff --git a/legacy_apps/back-to-back-detectors/inferserver/primary_detector_config.txt b/legacy_apps/back-to-back-detectors/inferserver/primary_detector_config.txt new file mode 100755 index 0000000..0e19437 --- /dev/null +++ b/legacy_apps/back-to-back-detectors/inferserver/primary_detector_config.txt @@ -0,0 +1,74 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 30 + backend { + inputs: [ { + name: "input_1" + }] + outputs: [ + {name: "output_cov/Sigmoid"}, + {name: "output_bbox/BiasAdd"} + ] + triton { + model_name: "Primary_Detector" + version: -1 + model_repo { + root: "../../../../../../samples/triton_tao_model_repo" + strict_model_config: true + } + } + } + preprocess { + network_format: MEDIA_FORMAT_NONE + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + channel_offsets: [0, 0, 0] + } + } + postprocess { + labelfile_path: "../../../../../../samples/models/Primary_Detector/labels.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/legacy_apps/back-to-back-detectors/inferserver/secondary_detector_config.txt b/legacy_apps/back-to-back-detectors/inferserver/secondary_detector_config.txt new file mode 100755 index 0000000..3dc3266 --- /dev/null +++ b/legacy_apps/back-to-back-detectors/inferserver/secondary_detector_config.txt @@ -0,0 +1,75 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +infer_config { + unique_id: 2 + gpu_ids: [0] + max_batch_size: 1 + backend { + inputs: [ { + name: "input_1" + }] + outputs: [ + {name: "output_bbox/BiasAdd"}, + {name: "output_cov/Sigmoid"} + ] + triton { + model_name: "facenet" + version: -1 + model_repo { + root: "../../../../../../samples/triton_tao_model_repo" + strict_model_config: true + } + } + } + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1" + maintain_aspect_ratio: 1 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + channel_offsets: [0, 0, 0] + } + } + postprocess { + labelfile_path: "../../../../../../samples/configs/tao_pretrained_models/labels_facenet.txt" + detection { + num_detected_classes: 1 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_class_ids: 2 + operate_on_gie_id: 1 + interval: 0 +} diff --git a/legacy_apps/back-to-back-detectors/primary_detector_config.txt b/legacy_apps/back-to-back-detectors/primary_detector_config.txt new file mode 100755 index 0000000..8b4ff75 --- /dev/null +++ b/legacy_apps/back-to-back-detectors/primary_detector_config.txt @@ -0,0 +1,72 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path, +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.00392156862745098 +onnx-file=/opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/resnet18_trafficcamnet_pruned.onnx +labelfile-path=/opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/labels.txt +int8-calib-file=/opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/cal_trt.bin +batch-size=1 +network-mode=2 +process-mode=1 +num-detected-classes=4 +interval=0 +gie-unique-id=1 +cluster-mode=2 +offsets=0.0;0.0;0.0 +maintain-aspect-ratio=0 + +[class-attrs-all] +pre-cluster-threshold=0.2 +topk=20 +nms-iou-threshold=0.5 diff --git a/legacy_apps/back-to-back-detectors/secondary_detector_config.txt b/legacy_apps/back-to-back-detectors/secondary_detector_config.txt new file mode 100755 index 0000000..85e578f --- /dev/null +++ b/legacy_apps/back-to-back-detectors/secondary_detector_config.txt @@ -0,0 +1,84 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path, +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +process-mode=2 +net-scale-factor=0.00392156862745098 +tlt-model-key=nvidia_tlt +tlt-encoded-model=../../../../../samples/models/tao_pretrained_models/facenet/model.etlt +labelfile-path=../../../../../samples/configs/tao_pretrained_models/labels_facenet.txt +int8-calib-file=../../../../../samples/models/tao_pretrained_models/facenet/int8_calibration.txt +model-engine-file=../../../../../samples/models/tao_pretrained_models/facenet/model.etlt_b1_gpu0_int8.engine +force-implicit-batch-dim=1 +batch-size=1 +network-mode=0 +num-detected-classes=1 +interval=0 +gie-unique-id=2 +infer-dims=3;416;736 +uff-input-blob-name=input_1 +output-blob-names=output_bbox/BiasAdd;output_cov/Sigmoid +input-object-min-width=64 +input-object-min-height=64 +maintain-aspect-ratio=1 +# Person has class-id 2 for the primary detector. This ensures that this secondary +# detector only works on persons. +operate-on-class-ids=2 +cluster-mode=2 + +[class-attrs-all] +pre-cluster-threshold=0.2 +topk=20 +nms-iou-threshold=0.5 diff --git a/legacy_apps/back-to-back-detectors/yolov8n-face.onnx b/legacy_apps/back-to-back-detectors/yolov8n-face.onnx new file mode 100644 index 0000000..b83edf3 Binary files /dev/null and b/legacy_apps/back-to-back-detectors/yolov8n-face.onnx differ diff --git a/legacy_apps/deepstream-occupancy-analytics/LICENSE.md b/legacy_apps/deepstream-occupancy-analytics/LICENSE.md new file mode 100755 index 0000000..8190858 --- /dev/null +++ b/legacy_apps/deepstream-occupancy-analytics/LICENSE.md @@ -0,0 +1,15 @@ +SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/legacy_apps/deepstream-occupancy-analytics/Makefile b/legacy_apps/deepstream-occupancy-analytics/Makefile new file mode 100755 index 0000000..85b5d96 --- /dev/null +++ b/legacy_apps/deepstream-occupancy-analytics/Makefile @@ -0,0 +1,86 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +APP:= deepstream-test5-analytics + +TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -) +DS_VER = $(shell deepstream-app -v | awk '$$1~/DeepStreamSDK/ {print substr($$2,1,3)}' ) + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream-$(DS_VER)/lib/ +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream-$(DS_VER)/bin/ + +ifeq ($(TARGET_DEVICE),aarch64) + CFLAGS:= -DPLATFORM_TEGRA +endif + +SRCS:= deepstream_test5_app_main.c +SRCS+= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-test5/deepstream_utc.c +SRCS+= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-app/deepstream_app.c +SRCS+= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-app/deepstream_app_config_parser.c +SRCS+= $(wildcard /opt/nvidia/deepstream/deepstream/sources/apps/apps-common/src/*.c) +SRCS+= /opt/nvidia/deepstream/deepstream/sources/libs/nvds_msgapi_common_src/nvds_utils.cpp + +INCS= $(wildcard *.h) +INC_DIR=/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-test5 + +PKGS:= gstreamer-1.0 gstreamer-video-1.0 x11 json-glib-1.0 + +OBJS:= $(SRCS:.c=.o) +OBJS:= $(OBJS:.cpp=.o) +OBJS+= deepstream_nvdsanalytics_meta.o + +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/apps/apps-common/includes -I./includes \ + -I/opt/nvidia/deepstream/deepstream/sources/includes \ + -I/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-app/ \ + -DDS_VERSION_MINOR=1 -DDS_VERSION_MAJOR=5 +CFLAGS+= -I$(INC_DIR) +CFLAGS+= -I/usr/local/cuda-$(CUDA_VER)/include + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvdsgst_helper \ + -lnvdsgst_customhelper -lnvdsgst_smartrecord -lnvds_utils -lnvds_msgbroker -lm \ + -lgstrtspserver-1.0 -ldl -Wl,-rpath,$(LIB_INSTALL_DIR) -lnvbufsurface -lnvds_logger -lcrypto +LIBS+= -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart + +CFLAGS+= `pkg-config --cflags $(PKGS)` + +LIBS+= `pkg-config --libs $(PKGS)` + +all: $(APP) + +deepstream_nvdsanalytics_meta.o: deepstream_nvdsanalytics_meta.cpp $(INCS) Makefile + $(CXX) -c -o $@ -Wall -Werror $(CFLAGS) $< + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +%.o: %.cpp $(INCS) Makefile + $(CXX) -c -o $@ -Wall -Werror $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CXX) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) + diff --git a/legacy_apps/deepstream-occupancy-analytics/README.md b/legacy_apps/deepstream-occupancy-analytics/README.md new file mode 100755 index 0000000..04138d0 --- /dev/null +++ b/legacy_apps/deepstream-occupancy-analytics/README.md @@ -0,0 +1,100 @@ +**People count application With Deepstream SDK and Transfer Learning Toolkit** + +* [Description](#description) +* [Prerequisites](#prerequisites) +* [Getting Started](#GettingStarted) +* [Build](#build) +* [Run](#run) +* [Output](#output) +* [References](#references) +

+ +

+ +## Description + + This is a sample application for counting people entering/leaving in a building using NVIDIA Deepstream SDK, Transfer Learning Toolkit (TLT) and pre-trained models. This application can be used to build real-time occupancy analytics application for smart buildings, hospitals, retail, etc. The application is based on deepstream-test5 sample application. + + It takes streaming video as input, counts the number of people crossing a tripwire and sends the live data to the cloud. In this application, you will learn: + + - How to use PeopleNet model from NGC + - How to use NvDsAnalytics plugin to draw line and count people crossing the line + - How to send the analytics data to cloud or another microservice over Kafka + + You can extend this application to change region of interest, use cloud-to-edge messaging to trigger record in the DeepStream application or build analytic dashboard or database to store the metadata. + +To learn how to build this demo step-by-step, check out the on-demand webinar on [Creating Intelligent places using DeepStream SDK](https://info.nvidia.com/iva-occupancy-webinar-reg-page.html?ondemandrgt=yes). + +## Prerequisites + + +- Install Deepstream: [https://docs.nvidia.com/metropolis/deepstream/dev-guide/index.html#page/DeepStream_Development_Guide/deepstream_quick_start.html#] + +- Download PeopleNet model: [https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet/files] + +- This application is based on deepstream-test5 application. More about test5 application: [https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_ref_app_test5.html] + +- Install Kafka: [https://kafka.apache.org/quickstart] and create the kafka topic: + + `tar -xzf kafka_2.13-3.5.0.tgz` + + `cd kafka_2.13-3.5.0` + + `bin/zookeeper-server-start.sh config/zookeeper.properties` + + `bin/kafka-server-start.sh config/server.properties` + + `bin/kafka-topics.sh --create --topic quickstart-events --bootstrap-server localhost:9092` + +## Getting Started + +- Preferably clone the repo in $DS_SDK_ROOT/sources/apps/sample_apps/ +- Download peoplnet model: `cd deepstream-occupancy-analytics/config && ./model.sh` +- For Jetson use: bin/jetson/libnvds_msgconv.so +- For x86 use: bin/x86/libnvds_msgconv.so + +## Build and Configure + +- Set CUDA_VER in the MakeFile as per platform. + + For Jetson, CUDA_VER=11.4 + + For x86, CUDA_VER=11.8 + + `cd deepstream-occupancy-analytics && make` + +- Set **msg-conv-msg2p-lib** at **[sink1]** group in + **dstest_occupancy_analytics.txt** as per platform + + For Jetson + + msg-conv-msg2p-lib=$DEEPSTREAM_SDK_PATH/deepstream-occupancy-analytics/bin/jetson/libnvds_msgconv.so + + For x86 + + msg-conv-msg2p-lib=$DEEPSTREAM_SDK_PATH/deepstream-occupancy-analytics/bin/x86/libnvds_msgconv.so + +## Run + + `./deepstream-test5-analytics -c config/dstest_occupancy_analytics.txt` + + In another terminal run this command to see the kafka messages: + + `bin/kafka-console-consumer.sh --topic quickstart-events --from-beginning --bootstrap-server localhost:9092` + + +## Output + + The output will look like this: + + ![alt-text](images/kafka_messages.gif) + + Where you can see the kafka messages for entry and exit count. + +## References + +- CREATE INTELLIGENT PLACES USING NVIDIA PRE-TRAINED VISION MODELS AND DEEPSTREAM SDK: [https://info.nvidia.com/iva-occupancy-webinar-reg-page.html?ondemandrgt=yes] +- Deepstream SDK: [https://developer.nvidia.com/deepstream-sdk] +- Deepstream Quick Start Guide: [https://docs.nvidia.com/metropolis/deepstream/dev-guide/index.html#page/DeepStream_Development_Guide/deepstream_quick_start.html#] +- Transfer Learning Toolkit: [https://developer.nvidia.com/transfer-learning-toolkit] + diff --git a/legacy_apps/deepstream-occupancy-analytics/config/dstest_occupancy_analytics.txt b/legacy_apps/deepstream-occupancy-analytics/config/dstest_occupancy_analytics.txt new file mode 100755 index 0000000..4d2ef91 --- /dev/null +++ b/legacy_apps/deepstream-occupancy-analytics/config/dstest_occupancy_analytics.txt @@ -0,0 +1,209 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[application] +enable-perf-measurement=1 +perf-measurement-interval-sec=5 +#gie-kitti-output-dir=streamscl + +[tiled-display] +enable=1 +rows=1 +columns=1 +width=1280 +height=720 +gpu-id=0 +#(0): nvbuf-mem-default - Default memory allocated, specific to particular platform +#(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla +#(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla +#(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla +#(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson +nvbuf-memory-type=0 + +[source0] +enable=1 +#Type - 1=CameraV4L2 2=URI 3=MultiURI +type=3 +uri=file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 +num-sources=1 +gpu-id=0 +nvbuf-memory-type=0 + +[source1] +enable=1 +#Type - 1=CameraV4L2 2=URI 3=MultiURI 4=RTSP +type=3 +uri=file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 +num-sources=1 +gpu-id=0 +nvbuf-memory-type=0 +# smart record specific fields, valid only for source type=4 +# 0 = disable, 1 = through cloud events, 2 = through cloud + local events +smart-record=2 +# 0 = mp4, 1 = mkv +#smart-rec-container=0 +smart-rec-start-time=1 +smart-rec-start-time=1 +smart-rec-file-prefix=smart_record +#smart-rec-dir-path=/home/monika/record +# cache size in seconds +smart-rec-cache=10 + +[sink0] +enable=1 +type=2 +#1=mp4 2=mkv./bin/kafka-topics --create --bootstrap-server localhost:9092 \ +#--replication-factor 1 --partitions 1 --topic users +container=1 +#1=h264 2=h265 +codec=1 +#encoder type 0=Hardware 1=Software +enc-type=0 +sync=0 +#iframeinterval=10 +bitrate=100000 +#H264 Profile - 0=Baseline 2=Main 4=High +#H265 Profile - 0=Main 1=Main10 +profile=0 +output-file=resnet.mp4 +source-id=0 + +[sink1] +enable=1 +#Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvoverlaysink 6=MsgConvBroker +type=6 +msg-conv-config=msgconv_sample_config.txt +# Name of library having custom implementation. +# msg-conv-msg2p-lib=/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-occupancy-analytics/bin/jetson/libnvds_msgconv.so +# msg-conv-msg2p-lib=/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-occupancy-analytics/bin/x86/libnvds_msgconv.so +#(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload +#(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal +#(256): PAYLOAD_RESERVED - Reserved type +#(257): PAYLOAD_CUSTOM - Custom schema payload +msg-conv-payload-type=0 +msg-broker-proto-lib=/opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so +#Provide your msg-broker-conn-str here +msg-broker-conn-str=localhost;9092;quickstart-events +#topic= +#Optional: +#msg-broker-config=../../deepstream-test4/cfg_kafka.txt + +[sink2] +enable=1 +type=1 +#1=mp4 2=mkv +container=1 +#1=h264 2=h265 3=mpeg4 +## only SW mpeg4 is supported right now. +codec=3 +sync=1 +bitrate=2000000 +output-file=out.mp4 +source-id=0 + +# sink type = 6 by default creates msg converter + broker. +# To use multiple brokers use this group for converter and use +# sink type = 6 with disable-msgconv = 1 +[message-converter] +enable=0 +msg-conv-config=msgconv_sample_config.txt +#(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload +#(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal +#(256): PAYLOAD_RESERVED - Reserved type +#(257): PAYLOAD_CUSTOM - Custom schema payload +msg-conv-payload-type=0 +# Id of component in case only selected message to parse. +#msg-conv-comp-id= + +# Configure this group to enable cloud message consumer. +[message-consumer0] +enable=0 +proto-lib=/opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so +conn-str=localhost;9092 +#config-file= +subscribe-topic-list=quickstart-events +# Use this option if message has sensor name as id instead of index (0,1,2 etc.). +sensor-list-file=msgconv_sample_config.txt + +[osd] +enable=1 +gpu-id=0 +border-width=1 +text-size=10 +text-color=1;1;1;1; +text-bg-color=0.3;0.3;0.3;1 +font=Arial +show-clock=0 +clock-x-offset=800 +clock-y-offset=820 +clock-text-size=12 +clock-color=1;0;0;0 +nvbuf-memory-type=0 + +[streammux] +gpu-id=0 +##Boolean property to inform muxer that sources are live +live-source=0 +batch-size=1 +##time out in usec, to wait after the first buffer is available +##to push the batch even if the complete batch is not formed +batched-push-timeout=40000 +## Set muxer output width and height +width=1920 +height=1080 +##Enable to maintain aspect ratio wrt source, and allow black borders, works +##along with width, height properties +enable-padding=0 +nvbuf-memory-type=0 +## If set to TRUE, system timestamp will be attached as ntp timestamp +## If set to FALSE, ntp timestamp from rtspsrc, if available, will be attached +# attach-sys-ts-as-ntp=1 + +[primary-gie] +enable=1 +gpu-id=0 +batch-size=1 +## 0=FP32, 1=INT8, 2=FP16 mode +bbox-border-color0=1;0;0;1 +#bbox-border-color1=0;1;1;1 +#bbox-border-color2=0;1;1;1 +#bbox-border-color3=0;1;0;1 +nvbuf-memory-type=0 +interval=0 +config-file=pgie_peoplenet_tao_config.txt +#infer-raw-output-dir=../../../../../samples/primary_detector_raw_output/ + +[tracker] +enable=1 +tracker-width=640 +tracker-height=384 +gpu-id=0 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +# ll-config-file required to set different tracker types +# ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_IOU.yml +# ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_NvSORT.yml +ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml +# ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml +# ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_NvDeepSORT.yml +enable-batch-process=0 + +[nvds-analytics] +enable=1 +config-file=nvdsanalytics_config.txt + +[tests] +file-loop=0 diff --git a/legacy_apps/deepstream-occupancy-analytics/config/model.sh b/legacy_apps/deepstream-occupancy-analytics/config/model.sh new file mode 100755 index 0000000..ce752ac --- /dev/null +++ b/legacy_apps/deepstream-occupancy-analytics/config/model.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +if [ ! -d peoplenet ];then + mkdir peoplenet +fi + +cd peoplenet +if [ ! -e labels.txt ];then + echo "Downloading peoplenet label.... " + wget https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet/versions/pruned_quantized_v2.3.2/files/labels.txt +fi + +if [ ! -e resnet34_peoplenet_pruned_int8.etlt ];then + echo "Downloading peoplenet etlt model.... " + wget https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet/versions/pruned_quantized_v2.3.2/files/resnet34_peoplenet_pruned_int8.etlt +fi + +if [ ! -e resnet34_peoplenet_pruned_int8.txt ];then + echo "Downloading peoplenet int8 .... " + wget https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet/versions/pruned_quantized_v2.3.2/files/resnet34_peoplenet_pruned_int8.txt +fi +cd - diff --git a/legacy_apps/deepstream-occupancy-analytics/config/msgconv_sample_config.txt b/legacy_apps/deepstream-occupancy-analytics/config/msgconv_sample_config.txt new file mode 100755 index 0000000..f27abd0 --- /dev/null +++ b/legacy_apps/deepstream-occupancy-analytics/config/msgconv_sample_config.txt @@ -0,0 +1,1942 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[sensor0] +enable=1 +type=Camera +id=HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor1] +enable=1 +type=Camera +id=HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor2] +enable=1 +type=Camera +id=HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor3] +enable=1 +type=Camera +id=HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor4] +enable=1 +type=Camera +id=HWY_20_AND_WACKER__WBA__4_11_2018_4_59_59_550_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor5] +enable=1 +type=Camera +id=HWY_20_AND_WACKER__EBA__4_11_2018_4_59_59_543_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor6] +enable=1 +type=Camera +id=HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor7] +enable=1 +type=Camera +id=HWY_20_AND_BRYANT__WB__4_11_2018_4_59_59_485_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor8] +enable=1 +type=Camera +id=HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor9] +enable=1 +type=Camera +id=HWY_20_AND_DEVON__EBA__4_11_2018_4_59_59_793_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor10] +enable=1 +type=Camera +id=HWY_20_AND_JFK__WB__4_11_2018_4_59_59_450_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor11] +enable=1 +type=Camera +id=HWY_20_AND_JFK__WBA__4_11_2018_4_59_59_860_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + + +[sensor12] +enable=1 +type=Camera +id=HWY_20_AND_JFK__EB__4_11_2018_4_59_59_872_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor13] +enable=1 +type=Camera +id=HWY_20_AND_UNIVERSITY__WB__4_11_2018_4_59_59_308_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor14] +enable=1 +type=Camera +id=HWY_20_AND_UNIVERSITY__EB__4_11_2018_4_59_59_734_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor15] +enable=1 +type=Camera +id=HWY_20_AND_CENTURY__WB__4_11_2018_5_00_00_072_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor16] +enable=1 +type=Camera +id=HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor17] +enable=1 +type=Camera +id=HWY_20_AND_WACKER__WB__4_11_2018_4_59_57_927_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor18] +enable=1 +type=Camera +id=HWY_20_AND_WACKER__EB__4_11_2018_4_59_59_473_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor19] +enable=1 +type=Camera +id=HWY_20_AND_LOCUST__HILL_EB_26-76__4_11_2018_4_59_59_433_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[place0] +enable=1 +id=0 +type=intersection/road +name=HWY_20_AND_LOCUST__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place1] +enable=1 +id=1 +type=intersection/road +name=HWY_20_AND_LOCUST__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place2] +enable=1 +id=2 +type=intersection/road +name=HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place3] +enable=1 +id=3 +type=intersection/road +name=HWY_20_AND_LOCUST +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place4] +enable=1 +id=4 +type=intersection/road +name=HWY_20_AND_WACKER__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place5] +enable=1 +id=5 +type=intersection/road +name=HWY_20_AND_WACKER__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place6] +enable=1 +id=6 +type=intersection/road +name=HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place7] +enable=1 +id=7 +type=intersection/road +name=HWY_20_AND_BRYANT__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place8] +enable=1 +id=8 +type=intersection/road +name=HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place9] +enable=1 +id=9 +type=intersection/road +name=HWY_20_AND_DEVON__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place10] +enable=1 +id=10 +type=intersection/road +name=HWY_20_AND_JFK__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place11] +enable=1 +id=11 +type=intersection/road +name=HWY_20_AND_JFK__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place12] +enable=1 +id=12 +type=intersection/road +name=HWY_20_AND_JFK__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place13] +enable=1 +id=13 +type=intersection/road +name=HWY_20_AND_UNIVERSITY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place14] +enable=1 +id=14 +type=intersection/road +name=HWY_20_AND_UNIVERSITY__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place15] +enable=1 +id=15 +type=intersection/road +name=HWY_20_AND_CENTURY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place16] +enable=1 +id=16 +type=intersection/road +name=HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place17] +enable=1 +id=17 +type=intersection/road +name=HWY_20_AND_WACKER__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place18] +enable=1 +id=18 +type=intersection/road +name=HWY_20_AND_WACKER__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place19] +enable=1 +id=19 +type=intersection/road +name=HWY_20_AND_LOCUST__HILL_EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +## Group 2 + +[sensor20] +enable=1 +type=Camera +id=INT2_HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor21] +enable=1 +type=Camera +id=INT2_HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor22] +enable=1 +type=Camera +id=INT2_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor23] +enable=1 +type=Camera +id=INT2_HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor24] +enable=1 +type=Camera +id=INT2_HWY_20_AND_WACKER__WBA__4_11_2018_4_59_59_550_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor25] +enable=1 +type=Camera +id=INT2_HWY_20_AND_WACKER__EBA__4_11_2018_4_59_59_543_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor26] +enable=1 +type=Camera +id=INT2_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor27] +enable=1 +type=Camera +id=INT2_HWY_20_AND_BRYANT__WB__4_11_2018_4_59_59_485_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor28] +enable=1 +type=Camera +id=INT2_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor29] +enable=1 +type=Camera +id=INT2_HWY_20_AND_DEVON__EBA__4_11_2018_4_59_59_793_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor30] +enable=1 +type=Camera +id=INT2_HWY_20_AND_JFK__WB__4_11_2018_4_59_59_450_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor31] +enable=1 +type=Camera +id=INT2_HWY_20_AND_JFK__WBA__4_11_2018_4_59_59_860_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + + +[sensor32] +enable=1 +type=Camera +id=INT2_HWY_20_AND_JFK__EB__4_11_2018_4_59_59_872_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor33] +enable=1 +type=Camera +id=INT2_HWY_20_AND_UNIVERSITY__WB__4_11_2018_4_59_59_308_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor34] +enable=1 +type=Camera +id=INT2_HWY_20_AND_UNIVERSITY__EB__4_11_2018_4_59_59_734_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor35] +enable=1 +type=Camera +id=INT2_HWY_20_AND_CENTURY__WB__4_11_2018_5_00_00_072_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor36] +enable=1 +type=Camera +id=INT2_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor37] +enable=1 +type=Camera +id=INT2_HWY_20_AND_WACKER__WB__4_11_2018_4_59_57_927_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor38] +enable=1 +type=Camera +id=INT2_HWY_20_AND_WACKER__EB__4_11_2018_4_59_59_473_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor39] +enable=1 +type=Camera +id=INT2_HWY_20_AND_LOCUST__HILL_EB_26-76__4_11_2018_4_59_59_433_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[place20] +enable=1 +id=0 +type=intersection/road +name=INT2_HWY_20_AND_LOCUST__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place21] +enable=1 +id=1 +type=intersection/road +name=INT2_HWY_20_AND_LOCUST__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place22] +enable=1 +id=2 +type=intersection/road +name=INT2_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place23] +enable=1 +id=3 +type=intersection/road +name=INT2_HWY_20_AND_LOCUST +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place24] +enable=1 +id=4 +type=intersection/road +name=INT2_HWY_20_AND_WACKER__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place25] +enable=1 +id=5 +type=intersection/road +name=INT2_HWY_20_AND_WACKER__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place26] +enable=1 +id=6 +type=intersection/road +name=INT2_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place27] +enable=1 +id=7 +type=intersection/road +name=INT2_HWY_20_AND_BRYANT__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place28] +enable=1 +id=8 +type=intersection/road +name=INT2_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place29] +enable=1 +id=9 +type=intersection/road +name=INT2_HWY_20_AND_DEVON__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place30] +enable=1 +id=10 +type=intersection/road +name=INT2_HWY_20_AND_JFK__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place31] +enable=1 +id=11 +type=intersection/road +name=INT2_HWY_20_AND_JFK__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place32] +enable=1 +id=12 +type=intersection/road +name=INT2_HWY_20_AND_JFK__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place33] +enable=1 +id=13 +type=intersection/road +name=INT2_HWY_20_AND_UNIVERSITY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place34] +enable=1 +id=14 +type=intersection/road +name=INT2_HWY_20_AND_UNIVERSITY__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place35] +enable=1 +id=15 +type=intersection/road +name=INT2_HWY_20_AND_CENTURY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place36] +enable=1 +id=16 +type=intersection/road +name=INT2_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place37] +enable=1 +id=17 +type=intersection/road +name=INT2_HWY_20_AND_WACKER__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place38] +enable=1 +id=18 +type=intersection/road +name=INT2_HWY_20_AND_WACKER__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place39] +enable=1 +id=19 +type=intersection/road +name=INT2_HWY_20_AND_LOCUST__HILL_EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +## Group 3 +[sensor40] +enable=1 +type=Camera +id=INT3_HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor41] +enable=1 +type=Camera +id=INT3_HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor42] +enable=1 +type=Camera +id=INT3_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor43] +enable=1 +type=Camera +id=INT3_HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor44] +enable=1 +type=Camera +id=INT3_HWY_20_AND_WACKER__WBA__4_11_2018_4_59_59_550_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor45] +enable=1 +type=Camera +id=INT3_HWY_20_AND_WACKER__EBA__4_11_2018_4_59_59_543_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor46] +enable=1 +type=Camera +id=INT3_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor47] +enable=1 +type=Camera +id=INT3_HWY_20_AND_BRYANT__WB__4_11_2018_4_59_59_485_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor48] +enable=1 +type=Camera +id=INT3_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor49] +enable=1 +type=Camera +id=INT3_HWY_20_AND_DEVON__EBA__4_11_2018_4_59_59_793_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor50] +enable=1 +type=Camera +id=INT3_HWY_20_AND_JFK__WB__4_11_2018_4_59_59_450_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor51] +enable=1 +type=Camera +id=INT3_HWY_20_AND_JFK__WBA__4_11_2018_4_59_59_860_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + + +[sensor52] +enable=1 +type=Camera +id=INT3_HWY_20_AND_JFK__EB__4_11_2018_4_59_59_872_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor53] +enable=1 +type=Camera +id=INT3_HWY_20_AND_UNIVERSITY__WB__4_11_2018_4_59_59_308_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor54] +enable=1 +type=Camera +id=INT3_HWY_20_AND_UNIVERSITY__EB__4_11_2018_4_59_59_734_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor55] +enable=1 +type=Camera +id=INT3_HWY_20_AND_CENTURY__WB__4_11_2018_5_00_00_072_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor56] +enable=1 +type=Camera +id=INT3_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor57] +enable=1 +type=Camera +id=INT3_HWY_20_AND_WACKER__WB__4_11_2018_4_59_57_927_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor58] +enable=1 +type=Camera +id=INT3_HWY_20_AND_WACKER__EB__4_11_2018_4_59_59_473_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor59] +enable=1 +type=Camera +id=INT3_HWY_20_AND_LOCUST__HILL_EB_26-76__4_11_2018_4_59_59_433_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[place40] +enable=1 +id=0 +type=intersection/road +name=INT3_HWY_20_AND_LOCUST__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place41] +enable=1 +id=1 +type=intersection/road +name=INT3_HWY_20_AND_LOCUST__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place42] +enable=1 +id=2 +type=intersection/road +name=INT3_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place43] +enable=1 +id=3 +type=intersection/road +name=INT3_HWY_20_AND_LOCUST +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place44] +enable=1 +id=4 +type=intersection/road +name=INT3_HWY_20_AND_WACKER__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place45] +enable=1 +id=5 +type=intersection/road +name=INT3_HWY_20_AND_WACKER__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place46] +enable=1 +id=6 +type=intersection/road +name=INT3_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place47] +enable=1 +id=7 +type=intersection/road +name=INT3_HWY_20_AND_BRYANT__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place48] +enable=1 +id=8 +type=intersection/road +name=INT3_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place49] +enable=1 +id=9 +type=intersection/road +name=INT3_HWY_20_AND_DEVON__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place50] +enable=1 +id=10 +type=intersection/road +name=INT3_HWY_20_AND_JFK__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place51] +enable=1 +id=11 +type=intersection/road +name=INT3_HWY_20_AND_JFK__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place52] +enable=1 +id=12 +type=intersection/road +name=INT3_HWY_20_AND_JFK__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place53] +enable=1 +id=13 +type=intersection/road +name=INT3_HWY_20_AND_UNIVERSITY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place54] +enable=1 +id=14 +type=intersection/road +name=INT3_HWY_20_AND_UNIVERSITY__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place55] +enable=1 +id=15 +type=intersection/road +name=INT3_HWY_20_AND_CENTURY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place56] +enable=1 +id=16 +type=intersection/road +name=INT3_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place57] +enable=1 +id=17 +type=intersection/road +name=INT3_HWY_20_AND_WACKER__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place58] +enable=1 +id=18 +type=intersection/road +name=INT3_HWY_20_AND_WACKER__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place59] +enable=1 +id=19 +type=intersection/road +name=INT3_HWY_20_AND_LOCUST__HILL_EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +## Group 4 +[sensor60] +enable=1 +type=Camera +id=INT4_HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor61] +enable=1 +type=Camera +id=INT4_HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor62] +enable=1 +type=Camera +id=INT4_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor63] +enable=1 +type=Camera +id=INT4_HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor64] +enable=1 +type=Camera +id=INT4_HWY_20_AND_WACKER__WBA__4_11_2018_4_59_59_550_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor65] +enable=1 +type=Camera +id=INT4_HWY_20_AND_WACKER__EBA__4_11_2018_4_59_59_543_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor66] +enable=1 +type=Camera +id=INT4_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor67] +enable=1 +type=Camera +id=INT4_HWY_20_AND_BRYANT__WB__4_11_2018_4_59_59_485_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor68] +enable=1 +type=Camera +id=INT4_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor69] +enable=1 +type=Camera +id=INT4_HWY_20_AND_DEVON__EBA__4_11_2018_4_59_59_793_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor70] +enable=1 +type=Camera +id=INT4_HWY_20_AND_JFK__WB__4_11_2018_4_59_59_450_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor71] +enable=1 +type=Camera +id=INT4_HWY_20_AND_JFK__WBA__4_11_2018_4_59_59_860_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + + +[sensor72] +enable=1 +type=Camera +id=INT4_HWY_20_AND_JFK__EB__4_11_2018_4_59_59_872_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor73] +enable=1 +type=Camera +id=INT4_HWY_20_AND_UNIVERSITY__WB__4_11_2018_4_59_59_308_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor74] +enable=1 +type=Camera +id=INT4_HWY_20_AND_UNIVERSITY__EB__4_11_2018_4_59_59_734_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor75] +enable=1 +type=Camera +id=INT4_HWY_20_AND_CENTURY__WB__4_11_2018_5_00_00_072_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor76] +enable=1 +type=Camera +id=INT4_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor77] +enable=1 +type=Camera +id=INT4_HWY_20_AND_WACKER__WB__4_11_2018_4_59_57_927_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor78] +enable=1 +type=Camera +id=INT4_HWY_20_AND_WACKER__EB__4_11_2018_4_59_59_473_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor79] +enable=1 +type=Camera +id=INT4_HWY_20_AND_LOCUST__HILL_EB_26-76__4_11_2018_4_59_59_433_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[place60] +enable=1 +id=0 +type=intersection/road +name=INT4_HWY_20_AND_LOCUST__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place61] +enable=1 +id=1 +type=intersection/road +name=INT4_HWY_20_AND_LOCUST__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place62] +enable=1 +id=2 +type=intersection/road +name=INT4_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place63] +enable=1 +id=3 +type=intersection/road +name=INT4_HWY_20_AND_LOCUST +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place64] +enable=1 +id=4 +type=intersection/road +name=INT4_HWY_20_AND_WACKER__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place65] +enable=1 +id=5 +type=intersection/road +name=INT4_HWY_20_AND_WACKER__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place66] +enable=1 +id=6 +type=intersection/road +name=INT4_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place67] +enable=1 +id=7 +type=intersection/road +name=INT4_HWY_20_AND_BRYANT__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place68] +enable=1 +id=8 +type=intersection/road +name=INT4_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place69] +enable=1 +id=9 +type=intersection/road +name=INT4_HWY_20_AND_DEVON__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place70] +enable=1 +id=10 +type=intersection/road +name=INT4_HWY_20_AND_JFK__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place71] +enable=1 +id=11 +type=intersection/road +name=INT4_HWY_20_AND_JFK__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place72] +enable=1 +id=12 +type=intersection/road +name=INT4_HWY_20_AND_JFK__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place73] +enable=1 +id=13 +type=intersection/road +name=INT4_HWY_20_AND_UNIVERSITY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place74] +enable=1 +id=14 +type=intersection/road +name=INT4_HWY_20_AND_UNIVERSITY__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place75] +enable=1 +id=15 +type=intersection/road +name=INT4_HWY_20_AND_CENTURY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place76] +enable=1 +id=16 +type=intersection/road +name=INT4_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place77] +enable=1 +id=17 +type=intersection/road +name=INT4_HWY_20_AND_WACKER__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place78] +enable=1 +id=18 +type=intersection/road +name=INT4_HWY_20_AND_WACKER__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place79] +enable=1 +id=19 +type=intersection/road +name=INT4_HWY_20_AND_LOCUST__HILL_EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +## Group 5 +[sensor80] +enable=1 +type=Camera +id=INT5_HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor81] +enable=1 +type=Camera +id=INT5_HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor82] +enable=1 +type=Camera +id=INT5_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor83] +enable=1 +type=Camera +id=INT5_HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor84] +enable=1 +type=Camera +id=INT5_HWY_20_AND_WACKER__WBA__4_11_2018_4_59_59_550_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor85] +enable=1 +type=Camera +id=INT5_HWY_20_AND_WACKER__EBA__4_11_2018_4_59_59_543_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor86] +enable=1 +type=Camera +id=INT5_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor87] +enable=1 +type=Camera +id=INT5_HWY_20_AND_BRYANT__WB__4_11_2018_4_59_59_485_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor88] +enable=1 +type=Camera +id=INT5_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor89] +enable=1 +type=Camera +id=INT5_HWY_20_AND_DEVON__EBA__4_11_2018_4_59_59_793_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor90] +enable=1 +type=Camera +id=INT5_HWY_20_AND_JFK__WB__4_11_2018_4_59_59_450_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor91] +enable=1 +type=Camera +id=INT5_HWY_20_AND_JFK__WBA__4_11_2018_4_59_59_860_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + + +[sensor92] +enable=1 +type=Camera +id=INT5_HWY_20_AND_JFK__EB__4_11_2018_4_59_59_872_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor93] +enable=1 +type=Camera +id=INT5_HWY_20_AND_UNIVERSITY__WB__4_11_2018_4_59_59_308_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor94] +enable=1 +type=Camera +id=INT5_HWY_20_AND_UNIVERSITY__EB__4_11_2018_4_59_59_734_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor95] +enable=1 +type=Camera +id=INT5_HWY_20_AND_CENTURY__WB__4_11_2018_5_00_00_072_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor96] +enable=1 +type=Camera +id=INT5_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor97] +enable=1 +type=Camera +id=INT5_HWY_20_AND_WACKER__WB__4_11_2018_4_59_57_927_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor98] +enable=1 +type=Camera +id=INT5_HWY_20_AND_WACKER__EB__4_11_2018_4_59_59_473_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor99] +enable=1 +type=Camera +id=INT5_HWY_20_AND_LOCUST__HILL_EB_26-76__4_11_2018_4_59_59_433_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[place80] +enable=1 +id=0 +type=intersection/road +name=INT5_HWY_20_AND_LOCUST__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place81] +enable=1 +id=1 +type=intersection/road +name=INT5_HWY_20_AND_LOCUST__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place82] +enable=1 +id=2 +type=intersection/road +name=INT5_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place83] +enable=1 +id=3 +type=intersection/road +name=INT5_HWY_20_AND_LOCUST +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place84] +enable=1 +id=4 +type=intersection/road +name=INT5_HWY_20_AND_WACKER__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place85] +enable=1 +id=5 +type=intersection/road +name=INT5_HWY_20_AND_WACKER__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place86] +enable=1 +id=6 +type=intersection/road +name=INT5_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place87] +enable=1 +id=7 +type=intersection/road +name=INT5_HWY_20_AND_BRYANT__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place88] +enable=1 +id=8 +type=intersection/road +name=INT5_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place89] +enable=1 +id=9 +type=intersection/road +name=INT5_HWY_20_AND_DEVON__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place90] +enable=1 +id=10 +type=intersection/road +name=INT5_HWY_20_AND_JFK__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place91] +enable=1 +id=11 +type=intersection/road +name=INT5_HWY_20_AND_JFK__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place92] +enable=1 +id=12 +type=intersection/road +name=INT5_HWY_20_AND_JFK__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place93] +enable=1 +id=13 +type=intersection/road +name=INT5_HWY_20_AND_UNIVERSITY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place94] +enable=1 +id=14 +type=intersection/road +name=INT5_HWY_20_AND_UNIVERSITY__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place95] +enable=1 +id=15 +type=intersection/road +name=INT5_HWY_20_AND_CENTURY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place96] +enable=1 +id=16 +type=intersection/road +name=INT5_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place97] +enable=1 +id=17 +type=intersection/road +name=INT5_HWY_20_AND_WACKER__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place98] +enable=1 +id=18 +type=intersection/road +name=INT5_HWY_20_AND_WACKER__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place99] +enable=1 +id=19 +type=intersection/road +name=INT5_HWY_20_AND_LOCUST__HILL_EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[analytics0] +enable=1 +id=XYZ +#description=Vehicle Detection and License Plate Recognition +source=OpenALR +version=1.0 + +[analytics1] +enable=1 +id=XYZ +#description=Vehicle Detection and License Plate Recognition 1 +source=OpenALR +version=1.0 + + diff --git a/legacy_apps/deepstream-occupancy-analytics/config/nvdsanalytics_config.txt b/legacy_apps/deepstream-occupancy-analytics/config/nvdsanalytics_config.txt new file mode 100755 index 0000000..37e3a50 --- /dev/null +++ b/legacy_apps/deepstream-occupancy-analytics/config/nvdsanalytics_config.txt @@ -0,0 +1,102 @@ +################################################################################ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-1] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=0 + + +[overcrowding-stream-1] +enable=0 +roi-OC=0;0;579;400;642;900;0;900 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +line-crossing-Exit=900;1000;850;900;300;1000;1350;800; +line-crossing-Entry=750;670;800;750;300;850;1350;650 +#line_color=0.75;0.25;0;1 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=balanced + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +line-crossing-Exit=900;1000;850;900;300;1000;1350;800; +line-crossing-Entry=750;670;800;750;300;850;1350;650 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=1 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=balanced + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 diff --git a/legacy_apps/deepstream-occupancy-analytics/config/pgie_peoplenet_tao_config.txt b/legacy_apps/deepstream-occupancy-analytics/config/pgie_peoplenet_tao_config.txt new file mode 100755 index 0000000..48d1e0d --- /dev/null +++ b/legacy_apps/deepstream-occupancy-analytics/config/pgie_peoplenet_tao_config.txt @@ -0,0 +1,53 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +tlt-model-key=tlt_encode +tlt-encoded-model=peoplenet/resnet34_peoplenet_pruned_int8.etlt +labelfile-path=peoplenet/labels.txt +# model-engine-file=peoplenet/resnet34_peoplenet_pruned_int8.etlt_b1_gpu0_int8.engine +int8-calib-file=peoplenet/resnet34_peoplenet_pruned_int8.txt +input-dims=3;544;960;0 +uff-input-blob-name=input_1 +batch-size=1 +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=1 +num-detected-classes=3 +cluster-mode=1 +interval=0 +gie-unique-id=1 +output-blob-names=output_bbox/BiasAdd;output_cov/Sigmoid + +[class-attrs-all] +pre-cluster-threshold=0.4 +## Set eps=0.7 and minBoxes for cluster-mode=1(DBSCAN) +eps=0.7 +minBoxes=1 + +[class-attrs-1] +pre-cluster-threshold=1.4 +## Set eps=0.7 and minBoxes for cluster-mode=1(DBSCAN) +eps=0.7 +minBoxes=1 +[class-attrs-2] +pre-cluster-threshold=1.4 +## Set eps=0.7 and minBoxes for cluster-mode=1(DBSCAN) +eps=0.7 +minBoxes=1 diff --git a/legacy_apps/deepstream-occupancy-analytics/deepstream_nvdsanalytics_meta.cpp b/legacy_apps/deepstream-occupancy-analytics/deepstream_nvdsanalytics_meta.cpp new file mode 100755 index 0000000..6ca15c1 --- /dev/null +++ b/legacy_apps/deepstream-occupancy-analytics/deepstream_nvdsanalytics_meta.cpp @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include "gstnvdsmeta.h" +#include "nvds_analytics_meta.h" +#include "analytics.h" + +/* custom_parse_nvdsanalytics_meta_data + * and extract nvanalytics metadata */ + extern "C" void +analytics_custom_parse_nvdsanalytics_meta_data (NvDsMetaList *l_user, AnalyticsUserMeta *data) +{ + std::stringstream out_string; + NvDsUserMeta *user_meta = (NvDsUserMeta *) l_user->data; + /* convert to metadata */ + NvDsAnalyticsFrameMeta *meta = + (NvDsAnalyticsFrameMeta *) user_meta->user_meta_data; + /* Fill the data for entry, exit,occupancy */ + data->lcc_cnt_entry = 0; + data->lcc_cnt_exit = 0; + data->lccum_cnt = 0; + data->lcc_cnt_entry = meta->objLCCumCnt["Entry"]; + data->lcc_cnt_exit = meta->objLCCumCnt["Exit"]; + + if (meta->objLCCumCnt["Entry"]> meta->objLCCumCnt["Exit"]) + data->lccum_cnt = meta->objLCCumCnt["Entry"] - meta->objLCCumCnt["Exit"]; + // g_print("Enter: %d, Exit: %d\n", data->lcc_cnt_entry,data->lcc_cnt_exit); +} + + diff --git a/legacy_apps/deepstream-occupancy-analytics/deepstream_test5_app_main.c b/legacy_apps/deepstream-occupancy-analytics/deepstream_test5_app_main.c new file mode 100755 index 0000000..f3a343e --- /dev/null +++ b/legacy_apps/deepstream-occupancy-analytics/deepstream_test5_app_main.c @@ -0,0 +1,1259 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "deepstream_app.h" +#include "deepstream_config_file_parser.h" +#include "nvds_version.h" + +#include +#include +#include + +#include "gstnvdsmeta.h" +#include "nvdsmeta_schema.h" + +#include "deepstream_test5_app.h" + +/*Analytics header*/ +#include "analytics.h" + +#define MAX_DISPLAY_LEN (64) +#define MAX_TIME_STAMP_LEN (64) +#define STREAMMUX_BUFFER_POOL_SIZE (16) + +/** @{ + * Macro's below and corresponding code-blocks are used to demonstrate + * nvmsgconv + Broker Metadata manipulation possibility + */ + +/** + * IMPORTANT Note 1: + * The code within the check for model_used == APP_CONFIG_ANALYTICS_RESNET_PGIE_3SGIE_TYPE_COLOR_MAKE + * is applicable as sample demo code for + * configs that use resnet PGIE model + * with class ID's: {0, 1, 2, 3} for {CAR, BICYCLE, PERSON, ROADSIGN} + * followed by optional Tracker + 3 X SGIEs (Vehicle-Type,Color,Make) + * only! + * Please comment out the code if using any other + * custom PGIE + SGIE combinations + * and use the code as reference to write your own + * NvDsEventMsgMeta generation code in generate_event_msg_meta() + * function + */ +typedef enum +{ + APP_CONFIG_ANALYTICS_MODELS_UNKNOWN = 0, + APP_CONFIG_ANALYTICS_RESNET_PGIE_3SGIE_TYPE_COLOR_MAKE = 1, +} AppConfigAnalyticsModel; + +#define RESNET10_PGIE_3SGIE_TYPE_COLOR_MAKECLASS_ID_CAR (0) +#ifdef GENERATE_DUMMY_META_EXT +#define RESNET10_PGIE_3SGIE_TYPE_COLOR_MAKECLASS_ID_PERSON (2) +#endif +/** @} */ + +/* PERSON ID definition. */ +#define PERSON_ID 0 + +#ifdef EN_DEBUG +#define LOGD(...) printf(__VA_ARGS__) +#else +#define LOGD(...) +#endif + +static TestAppCtx *testAppCtx; +GST_DEBUG_CATEGORY (NVDS_APP); + +/** @{ imported from deepstream-app as is */ + + +#define MAX_INSTANCES 128 +#define APP_TITLE "DeepStreamTest5App" + +#define DEFAULT_X_WINDOW_WIDTH 1920 +#define DEFAULT_X_WINDOW_HEIGHT 1080 + +AppCtx *appCtx[MAX_INSTANCES]; +static guint cintr = FALSE; +static GMainLoop *main_loop = NULL; +static gchar **cfg_files = NULL; +static gchar **input_files = NULL; +static gchar **override_cfg_file = NULL; +static gboolean playback_utc = TRUE; +static gboolean print_version = FALSE; +static gboolean show_bbox_text = TRUE; +static gboolean force_tcp = TRUE; +static gboolean print_dependencies_version = FALSE; +static gboolean quit = FALSE; +static gint return_value = 0; +static guint num_instances; +static guint num_input_files; +static GMutex fps_lock; +static gdouble fps[MAX_SOURCE_BINS]; +static gdouble fps_avg[MAX_SOURCE_BINS]; + +static Display *display = NULL; +static Window windows[MAX_INSTANCES] = { 0 }; + +static GThread *x_event_thread = NULL; +static GMutex disp_lock; + +static guint rrow, rcol, rcfg; +static gboolean rrowsel = FALSE, selecting = FALSE; +static AppConfigAnalyticsModel model_used = APP_CONFIG_ANALYTICS_MODELS_UNKNOWN; + +/** @} imported from deepstream-app as is */ +GOptionEntry entries[] = { + {"version", 'v', 0, G_OPTION_ARG_NONE, &print_version, + "Print DeepStreamSDK version", NULL} + , + {"tiledtext", 't', 0, G_OPTION_ARG_NONE, &show_bbox_text, + "Display Bounding box labels in tiled mode", NULL} + , + {"version-all", 0, 0, G_OPTION_ARG_NONE, &print_dependencies_version, + "Print DeepStreamSDK and dependencies version", NULL} + , + {"cfg-file", 'c', 0, G_OPTION_ARG_FILENAME_ARRAY, &cfg_files, + "Set the config file", NULL} + , + {"override-cfg-file", 'o', 0, G_OPTION_ARG_FILENAME_ARRAY, &override_cfg_file, + "Set the override config file, used for on-the-fly model update feature", + NULL} + , + {"input-file", 'i', 0, G_OPTION_ARG_FILENAME_ARRAY, &input_files, + "Set the input file", NULL} + , + {"playback-utc", 'p', 0, G_OPTION_ARG_INT, &playback_utc, + "Playback utc; default=true (base UTC from file/rtsp URL); =false (base UTC from file-URL or RTCP Sender Report)", + NULL} + , + {"pgie-model-used", 'm', 0, G_OPTION_ARG_INT, &model_used, + "PGIE Model used; {0 - Unknown [DEFAULT]}, {1: Resnet 4-class [Car, Bicycle, Person, Roadsign]}", + NULL} + , + {"no-force-tcp", 0, G_OPTION_FLAG_REVERSE, G_OPTION_ARG_NONE, &force_tcp, + "Do not force TCP for RTP transport", NULL} + , + {NULL} + , +}; + + + static void +generate_ts_rfc3339 (char *buf, int buf_size) +{ + time_t tloc; + struct tm tm_log; + struct timespec ts; + char strmsec[6]; //.nnnZ\0 + + clock_gettime (CLOCK_REALTIME, &ts); + memcpy (&tloc, (void *) (&ts.tv_sec), sizeof (time_t)); + gmtime_r (&tloc, &tm_log); + strftime (buf, buf_size, "%Y-%m-%dT%H:%M:%S", &tm_log); + int ms = ts.tv_nsec / 1000000; + g_snprintf (strmsec, sizeof (strmsec), ".%.3dZ", ms); + strncat (buf, strmsec, buf_size); +} + + + + static GstClockTime +generate_ts_rfc3339_from_ts (char *buf, int buf_size, GstClockTime ts, + gchar * src_uri, gint stream_id) +{ + time_t tloc; + struct tm tm_log; + char strmsec[6]; //.nnnZ\0 + int ms; + + GstClockTime ts_generated; + + if (playback_utc + || (appCtx[0]->config.multi_source_config[stream_id].type != + NV_DS_SOURCE_RTSP)) { + if (testAppCtx->streams[stream_id].meta_number == 0) { + testAppCtx->streams[stream_id].timespec_first_frame = + extract_utc_from_uri (src_uri); + memcpy (&tloc, + (void *) (&testAppCtx->streams[stream_id].timespec_first_frame. + tv_sec), sizeof (time_t)); + ms = testAppCtx->streams[stream_id].timespec_first_frame.tv_nsec / + 1000000; + testAppCtx->streams[stream_id].gst_ts_first_frame = ts; + ts_generated = + GST_TIMESPEC_TO_TIME (testAppCtx->streams[stream_id]. + timespec_first_frame); + if (ts_generated == 0) { + g_print + ("WARNING; playback mode used with URI [%s] not conforming to timestamp format;" + " check README; using system-time\n", src_uri); + clock_gettime (CLOCK_REALTIME, + &testAppCtx->streams[stream_id].timespec_first_frame); + ts_generated = + GST_TIMESPEC_TO_TIME (testAppCtx->streams[stream_id]. + timespec_first_frame); + } + } else { + GstClockTime ts_current = + GST_TIMESPEC_TO_TIME (testAppCtx-> + streams[stream_id].timespec_first_frame) + (ts - + testAppCtx->streams[stream_id].gst_ts_first_frame); + struct timespec timespec_current; + GST_TIME_TO_TIMESPEC (ts_current, timespec_current); + memcpy (&tloc, (void *) (×pec_current.tv_sec), sizeof (time_t)); + ms = timespec_current.tv_nsec / 1000000; + ts_generated = ts_current; + } + } else { + /** ts itself is UTC Time in ns */ + struct timespec timespec_current; + GST_TIME_TO_TIMESPEC (ts, timespec_current); + memcpy (&tloc, (void *) (×pec_current.tv_sec), sizeof (time_t)); + ms = timespec_current.tv_nsec / 1000000; + ts_generated = ts; + } + gmtime_r (&tloc, &tm_log); + strftime (buf, buf_size, "%Y-%m-%dT%H:%M:%S", &tm_log); + g_snprintf (strmsec, sizeof (strmsec), ".%.3dZ", ms); + strncat (buf, strmsec, buf_size); + LOGD ("ts=%s\n", buf); + + return ts_generated; +} + + + static gpointer +meta_copy_func (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + NvDsEventMsgMeta *dstMeta = NULL; + + dstMeta = g_memdup (srcMeta, sizeof (NvDsEventMsgMeta)); + + if (srcMeta->ts) + dstMeta->ts = g_strdup (srcMeta->ts); + + if (srcMeta->objSignature.size > 0) { + dstMeta->objSignature.signature = g_memdup (srcMeta->objSignature.signature, + srcMeta->objSignature.size); + dstMeta->objSignature.size = srcMeta->objSignature.size; + } + + if (srcMeta->objectId) { + dstMeta->objectId = g_strdup (srcMeta->objectId); + } + + /* + if (srcMeta->sensorStr) { + dstMeta->sensorStr = g_strdup (srcMeta->sensorStr); + } + */ + + if (srcMeta->extMsgSize > 0) { + if (srcMeta->objType == NVDS_OBJECT_TYPE_PERSON) { + NvDsPersonObject *srcObj = (NvDsPersonObject *) srcMeta->extMsg; + NvDsPersonObject *obj = + (NvDsPersonObject *) g_malloc0 (sizeof (NvDsPersonObject)); + + obj->age = srcObj->age; + + if (srcObj->gender) + obj->gender = g_strdup (srcObj->gender); + if (srcObj->cap) + obj->cap = g_strdup (srcObj->cap); + if (srcObj->hair) + obj->hair = g_strdup (srcObj->hair); + if (srcObj->apparel) + obj->apparel = g_strdup (srcObj->apparel); + + dstMeta->extMsg = obj; + dstMeta->extMsgSize = sizeof (NvDsPersonObject); + } + } + + return dstMeta; +} + + static void +meta_free_func (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + user_meta->user_meta_data = NULL; + + if (srcMeta->ts) { + g_free (srcMeta->ts); + } + + if (srcMeta->objSignature.size > 0) { + g_free (srcMeta->objSignature.signature); + srcMeta->objSignature.size = 0; + } + + if (srcMeta->objectId) { + g_free (srcMeta->objectId); + } + + if (srcMeta->sensorStr) { + g_free (srcMeta->sensorStr); + } + + if (srcMeta->extMsgSize > 0) { + if (srcMeta->objType == NVDS_OBJECT_TYPE_VEHICLE) { + NvDsVehicleObject *obj = (NvDsVehicleObject *) srcMeta->extMsg; + if (obj->type) + g_free (obj->type); + if (obj->color) + g_free (obj->color); + if (obj->make) + g_free (obj->make); + if (obj->model) + g_free (obj->model); + if (obj->license) + g_free (obj->license); + if (obj->region) + g_free (obj->region); + } else if (srcMeta->objType == NVDS_OBJECT_TYPE_PERSON) { + NvDsPersonObject *obj = (NvDsPersonObject *) srcMeta->extMsg; + + if (obj->gender) + g_free (obj->gender); + if (obj->cap) + g_free (obj->cap); + if (obj->hair) + g_free (obj->hair); + if (obj->apparel) + g_free (obj->apparel); + } + g_free (srcMeta->extMsg); + srcMeta->extMsg = NULL; + srcMeta->extMsgSize = 0; + } + g_free (srcMeta); +} + +#ifdef GENERATE_DUMMY_META_EXT + +#endif /**< GENERATE_DUMMY_META_EXT */ + + static void +generate_event_msg_meta (gpointer data, gint class_id, gboolean useTs, + GstClockTime ts, gchar * src_uri, gint stream_id, guint sensor_id, + AnalyticsUserMeta * obj_params, float scaleW, float scaleH, + NvDsFrameMeta * frame_meta) +{ + NvDsEventMsgMeta *meta = (NvDsEventMsgMeta *) data; + GstClockTime ts_generated = 0; + meta->objType = NVDS_OBJECT_TYPE_UNKNOWN; /**< object unknown */ + meta->frameId = frame_meta->frame_num; + meta->ts = (gchar *) g_malloc0 (MAX_TIME_STAMP_LEN + 1); + meta->objectId = (gchar *) g_malloc0 (MAX_LABEL_SIZE); + + //strncpy (meta->objectId, obj_params->obj_label, MAX_LABEL_SIZE); + + /** INFO: This API is called once for every 30 frames (now) */ + if (useTs && src_uri) { + ts_generated = + generate_ts_rfc3339_from_ts (meta->ts, MAX_TIME_STAMP_LEN, ts, src_uri, + stream_id); + } else { + generate_ts_rfc3339 (meta->ts, MAX_TIME_STAMP_LEN); + } + + /** tracking ID */ + meta->trackingId = class_id; + + (void) ts_generated; + meta->type = NVDS_EVENT_ENTRY; + meta->objType = NVDS_OBJECT_TYPE_PERSON; + meta->objClassId = PERSON_ID; + meta->occupancy = obj_params->lccum_cnt; + meta->lccum_cnt_entry = obj_params->lcc_cnt_entry; + meta->lccum_cnt_exit = obj_params->lcc_cnt_exit ; + meta->source_id = obj_params->source_id; +// g_print("source id: %d, Enter: %d, Exit: %d\n", meta->source_id, meta->lccum_cnt_entry, meta->lccum_cnt_exit); + +} + +/* + * Access analytics data. + */ +void analytics_custom_parse_nvdsanalytics_meta_data (NvDsMetaList *l_user, AnalyticsUserMeta *data); + +/** + * Callback function to be called once all inferences (Primary + Secondary) + * are done. This is opportunity to modify content of the metadata. + * e.g. Here Person is being replaced with Man/Woman and corresponding counts + * are being maintained. It should be modified according to network classes + * or can be removed altogether if not required. + */ + static void +bbox_generated_probe_after_analytics (AppCtx * appCtx, GstBuffer * buf, + NvDsBatchMeta * batch_meta, guint index) +{ + NvDsObjectMeta *obj_meta = NULL; + GstClockTime buffer_pts = 0; + guint32 stream_id = 0; + + + if (!appCtx->config.dsanalytics_config.enable){ + g_print ("Unable to get nvdsanalytics src pad\n"); + return; + } + + for (NvDsMetaList * l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = l_frame->data; + stream_id = frame_meta->source_id; + GstClockTime buf_ntp_time = 0; + if (playback_utc == FALSE) { + /** Calculate the buffer-NTP-time + * derived from this stream's RTCP Sender Report here: + */ + StreamSourceInfo *src_stream = &testAppCtx->streams[stream_id]; + buf_ntp_time = frame_meta->ntp_timestamp; + + if (buf_ntp_time < src_stream->last_ntp_time) { + NVGSTDS_WARN_MSG_V ("Source %d: NTP timestamps are backward in time." + " Current: %lu previous: %lu", stream_id, buf_ntp_time, + src_stream->last_ntp_time); + } + src_stream->last_ntp_time = buf_ntp_time; + } + + GList *l; + NvDsMetaList *l_analyticsuser; + l_analyticsuser = frame_meta->frame_user_meta_list; + + AnalyticsUserMeta *user_data = + (AnalyticsUserMeta *) g_malloc0(sizeof(AnalyticsUserMeta)); + if (l_analyticsuser != NULL) { + analytics_custom_parse_nvdsanalytics_meta_data(l_analyticsuser, user_data); + } + user_data->source_id = stream_id; + + // l_analyticsuser = l_analyticsuser->next; + + /* Code from test5 application */ + for (l = frame_meta->obj_meta_list; l != NULL; l = l->next) { + /* Now using above information we need to form a text that should + * be displayed on top of the bounding box, so lets form it here. */ + obj_meta = (NvDsObjectMeta *) (l->data); + + { + /** + * Enable only if this callback is after tiler + * NOTE: Scaling back code-commented + * now that bbox_generated_probe_after_analytics() is post analytics + * (say pgie, tracker or sgie) + * and before tiler, no plugin shall scale metadata and will be + * corresponding to the nvstreammux resolution + */ + float scaleW = 0; + float scaleH = 0; + /* Frequency of messages to be send will be based on use case. + * Here message is being sent for first object every 30 frames. + */ + buffer_pts = frame_meta->buf_pts; + if (!appCtx->config.streammux_config.pipeline_width + || !appCtx->config.streammux_config.pipeline_height) { + g_print ("invalid pipeline params\n"); + return; + } + LOGD ("stream %d==%d [%d X %d]\n", frame_meta->source_id, + frame_meta->pad_index, frame_meta->source_frame_width, + frame_meta->source_frame_height); + scaleW = + (float) frame_meta->source_frame_width / + appCtx->config.streammux_config.pipeline_width; + scaleH = + (float) frame_meta->source_frame_height / + appCtx->config.streammux_config.pipeline_height; + + if (playback_utc == FALSE) { + /** Use the buffer-NTP-time derived from this stream's RTCP Sender + * Report here: + */ + buffer_pts = buf_ntp_time; + } + /** Generate NvDsEventMsgMeta for every object */ + NvDsEventMsgMeta *msg_meta = + (NvDsEventMsgMeta *) g_malloc0 (sizeof (NvDsEventMsgMeta)); + generate_event_msg_meta (msg_meta, PERSON_ID, TRUE, + /**< useTs NOTE: Pass FALSE for files without base-timestamp in URI */ + buffer_pts, + appCtx->config.multi_source_config[stream_id].uri, stream_id, + appCtx->config.multi_source_config[stream_id].camera_id, + user_data, scaleW, scaleH, frame_meta); + testAppCtx->streams[stream_id].meta_number++; + NvDsUserMeta *user_event_meta = + nvds_acquire_user_meta_from_pool (batch_meta); + if (user_event_meta) { + /* + * Since generated event metadata has custom objects for + * Vehicle / Person which are allocated dynamically, we are + * setting copy and free function to handle those fields when + * metadata copy happens between two components. + */ + user_event_meta->user_meta_data = (void *) msg_meta; + user_event_meta->base_meta.batch_meta = batch_meta; + user_event_meta->base_meta.meta_type = NVDS_EVENT_MSG_META; + user_event_meta->base_meta.copy_func = + (NvDsMetaCopyFunc) meta_copy_func; + user_event_meta->base_meta.release_func = + (NvDsMetaReleaseFunc) meta_free_func; + nvds_add_user_meta_to_frame (frame_meta, user_event_meta); + } else { + g_print ("Error in attaching event meta to buffer\n"); + } + } + } + testAppCtx->streams[stream_id].frameCount++; + + g_free(user_data); + } +} + +/** @{ imported from deepstream-app as is */ + +/** + * Function to handle program interrupt signal. + * It installs default handler after handling the interrupt. + */ + static void +_intr_handler (int signum) +{ + struct sigaction action; + + NVGSTDS_ERR_MSG_V ("User Interrupted.. \n"); + + memset (&action, 0, sizeof (action)); + action.sa_handler = SIG_DFL; + + sigaction (SIGINT, &action, NULL); + + cintr = TRUE; +} + +/** + * callback function to print the performance numbers of each stream. + */ + static void +perf_cb (gpointer context, NvDsAppPerfStruct * str) +{ + static guint header_print_cnt = 0; + guint i; + AppCtx *appCtx = (AppCtx *) context; + guint numf = str->num_instances; + + g_mutex_lock (&fps_lock); + for (i = 0; i < numf; i++) { + fps[i] = str->fps[i]; + fps_avg[i] = str->fps_avg[i]; + } + + if (header_print_cnt % 20 == 0) { + g_print ("\n**PERF: "); + for (i = 0; i < numf; i++) { + g_print ("FPS %d (Avg)\t", i); + } + g_print ("\n"); + header_print_cnt = 0; + } + header_print_cnt++; + + time_t t = time (NULL); + struct tm *tm = localtime (&t); + printf ("%s", asctime (tm)); + if (num_instances > 1) + g_print ("PERF(%d): ", appCtx->index); + else + g_print ("**PERF: "); + + for (i = 0; i < numf; i++) { + g_print ("%.2f (%.2f)\t", fps[i], fps_avg[i]); + } + g_print ("\n"); + g_mutex_unlock (&fps_lock); +} + +/** + * Loop function to check the status of interrupts. + * It comes out of loop if application got interrupted. + */ + static gboolean +check_for_interrupt (gpointer data) +{ + if (quit) { + return FALSE; + } + + if (cintr) { + cintr = FALSE; + + quit = TRUE; + g_main_loop_quit (main_loop); + + return FALSE; + } + return TRUE; +} + +/* + * Function to install custom handler for program interrupt signal. + */ + static void +_intr_setup (void) +{ + struct sigaction action; + + memset (&action, 0, sizeof (action)); + action.sa_handler = _intr_handler; + + sigaction (SIGINT, &action, NULL); +} + + static gboolean +kbhit (void) +{ + struct timeval tv; + fd_set rdfs; + + tv.tv_sec = 0; + tv.tv_usec = 0; + + FD_ZERO (&rdfs); + FD_SET (STDIN_FILENO, &rdfs); + + select (STDIN_FILENO + 1, &rdfs, NULL, NULL, &tv); + return FD_ISSET (STDIN_FILENO, &rdfs); +} + +/* + * Function to enable / disable the canonical mode of terminal. + * In non canonical mode input is available immediately (without the user + * having to type a line-delimiter character). + */ + static void +changemode (int dir) +{ + static struct termios oldt, newt; + + if (dir == 1) { + tcgetattr (STDIN_FILENO, &oldt); + newt = oldt; + newt.c_lflag &= ~(ICANON); + tcsetattr (STDIN_FILENO, TCSANOW, &newt); + } else + tcsetattr (STDIN_FILENO, TCSANOW, &oldt); +} + + static void +print_runtime_commands (void) +{ + g_print ("\nRuntime commands:\n" + "\th: Print this help\n" + "\tq: Quit\n\n" "\tp: Pause\n" "\tr: Resume\n\n"); + + if (appCtx[0]->config.tiled_display_config.enable) { + g_print + ("NOTE: To expand a source in the 2D tiled display and view object details," + " left-click on the source.\n" + " To go back to the tiled display, right-click anywhere on the window.\n\n"); + } +} + +/** + * Loop function to check keyboard inputs and status of each pipeline. + */ + static gboolean +event_thread_func (gpointer arg) +{ + guint i; + gboolean ret = TRUE; + + // Check if all instances have quit + for (i = 0; i < num_instances; i++) { + if (!appCtx[i]->quit) + break; + } + + if (i == num_instances) { + quit = TRUE; + g_main_loop_quit (main_loop); + return FALSE; + } + // Check for keyboard input + if (!kbhit ()) { + //continue; + return TRUE; + } + int c = fgetc (stdin); + g_print ("\n"); + + gint source_id; + GstElement *tiler = appCtx[rcfg]->pipeline.tiled_display_bin.tiler; + g_object_get (G_OBJECT (tiler), "show-source", &source_id, NULL); + + if (selecting) { + if (rrowsel == FALSE) { + if (c >= '0' && c <= '9') { + rrow = c - '0'; + g_print ("--selecting source row %d--\n", rrow); + rrowsel = TRUE; + } + } else { + if (c >= '0' && c <= '9') { + int tile_num_columns = appCtx[rcfg]->config.tiled_display_config.columns; + rcol = c - '0'; + selecting = FALSE; + rrowsel = FALSE; + source_id = tile_num_columns * rrow + rcol; + g_print ("--selecting source col %d sou=%d--\n", rcol, source_id); + if (source_id >= (gint) appCtx[rcfg]->config.num_source_sub_bins) { + source_id = -1; + } else { + appCtx[rcfg]->show_bbox_text = TRUE; + appCtx[rcfg]->active_source_index = source_id; + g_object_set (G_OBJECT (tiler), "show-source", source_id, NULL); + } + } + } + } + switch (c) { + case 'h': + print_runtime_commands (); + break; + case 'p': + for (i = 0; i < num_instances; i++) + pause_pipeline (appCtx[i]); + break; + case 'r': + for (i = 0; i < num_instances; i++) + resume_pipeline (appCtx[i]); + break; + case 'q': + quit = TRUE; + g_main_loop_quit (main_loop); + ret = FALSE; + break; + case 'c': + if (selecting == FALSE && source_id == -1) { + g_print("--selecting config file --\n"); + c = fgetc(stdin); + if (c >= '0' && c <= '9') { + rcfg = c - '0'; + if (rcfg < num_instances) { + g_print("--selecting config %d--\n", rcfg); + } else { + g_print("--selected config file %d out of bound, reenter\n", rcfg); + rcfg = 0; + } + } + } + break; + case 'z': + if (source_id == -1 && selecting == FALSE) { + g_print ("--selecting source --\n"); + selecting = TRUE; + } else { + if (!show_bbox_text) { + GstElement *nvosd = appCtx[rcfg]->pipeline.instance_bins[0].osd_bin.nvosd; + g_object_set (G_OBJECT (nvosd), "display-text", FALSE, NULL); + g_object_set (G_OBJECT (tiler), "show-source", -1, NULL); + } + appCtx[rcfg]->active_source_index = -1; + selecting = FALSE; + rcfg = 0; + g_print("--tiled mode --\n"); + } + break; + default: + break; + } + return ret; +} + + static int +get_source_id_from_coordinates (float x_rel, float y_rel, AppCtx *appCtx) +{ + int tile_num_rows = appCtx->config.tiled_display_config.rows; + int tile_num_columns = appCtx->config.tiled_display_config.columns; + + int source_id = (int) (x_rel * tile_num_columns); + source_id += ((int) (y_rel * tile_num_rows)) * tile_num_columns; + + /* Don't allow clicks on empty tiles. */ + if (source_id >= (gint) appCtx->config.num_source_sub_bins) + source_id = -1; + + return source_id; +} + +/** + * Thread to monitor X window events. + */ + static gpointer +nvds_x_event_thread (gpointer data) +{ + g_mutex_lock (&disp_lock); + while (display) { + XEvent e; + guint index; + while (XPending (display)) { + XNextEvent (display, &e); + switch (e.type) { + case ButtonPress: + { + XWindowAttributes win_attr; + XButtonEvent ev = e.xbutton; + gint source_id; + GstElement *tiler; + + XGetWindowAttributes (display, ev.window, &win_attr); + + for (index = 0; index < MAX_INSTANCES; index++) + if (ev.window == windows[index]) + break; + + tiler = appCtx[index]->pipeline.tiled_display_bin.tiler; + g_object_get (G_OBJECT (tiler), "show-source", &source_id, NULL); + + if (ev.button == Button1 && source_id == -1) { + source_id = + get_source_id_from_coordinates (ev.x * 1.0 / win_attr.width, + ev.y * 1.0 / win_attr.height, appCtx[index]); + if (source_id > -1) { + g_object_set (G_OBJECT (tiler), "show-source", source_id, NULL); + appCtx[index]->active_source_index = source_id; + appCtx[index]->show_bbox_text = TRUE; + GstElement *nvosd = appCtx[index]->pipeline.instance_bins[0].osd_bin.nvosd; + g_object_set (G_OBJECT (nvosd), "display-text", TRUE, NULL); + } + } else if (ev.button == Button3) { + g_object_set (G_OBJECT (tiler), "show-source", -1, NULL); + appCtx[index]->active_source_index = -1; + if (!show_bbox_text) { + appCtx[index]->show_bbox_text = FALSE; + GstElement *nvosd = appCtx[index]->pipeline.instance_bins[0].osd_bin.nvosd; + g_object_set (G_OBJECT (nvosd), "display-text", FALSE, NULL); + } + } + } + break; + case KeyRelease: + { + KeySym p, r, q; + guint i; + p = XKeysymToKeycode (display, XK_P); + r = XKeysymToKeycode (display, XK_R); + q = XKeysymToKeycode (display, XK_Q); + if (e.xkey.keycode == p) { + for (i = 0; i < num_instances; i++) + pause_pipeline (appCtx[i]); + break; + } + if (e.xkey.keycode == r) { + for (i = 0; i < num_instances; i++) + resume_pipeline (appCtx[i]); + break; + } + if (e.xkey.keycode == q) { + quit = TRUE; + g_main_loop_quit (main_loop); + } + } + break; + case ClientMessage: + { + Atom wm_delete; + for (index = 0; index < MAX_INSTANCES; index++) + if (e.xclient.window == windows[index]) + break; + + wm_delete = XInternAtom (display, "WM_DELETE_WINDOW", 1); + if (wm_delete != None && wm_delete == (Atom) e.xclient.data.l[0]) { + quit = TRUE; + g_main_loop_quit (main_loop); + } + } + break; + } + } + g_mutex_unlock (&disp_lock); + g_usleep (G_USEC_PER_SEC / 20); + g_mutex_lock (&disp_lock); + } + g_mutex_unlock (&disp_lock); + return NULL; +} + +/** + * callback function to add application specific metadata. + * Here it demonstrates how to display the URI of source in addition to + * the text generated after inference. + */ + static gboolean +overlay_graphics (AppCtx * appCtx, GstBuffer * buf, + NvDsBatchMeta * batch_meta, guint index) +{ + return TRUE; +} + +/** @} imported from deepstream-app as is */ + + int +main (int argc, char *argv[]) +{ + testAppCtx = (TestAppCtx *) g_malloc0 (sizeof (TestAppCtx)); + GOptionContext *ctx = NULL; + GOptionGroup *group = NULL; + GError *error = NULL; + guint i; + + ctx = g_option_context_new ("Nvidia DeepStream Test5"); + group = g_option_group_new ("abc", NULL, NULL, NULL, NULL); + g_option_group_add_entries (group, entries); + + g_option_context_set_main_group (ctx, group); + g_option_context_add_group (ctx, gst_init_get_option_group ()); + + GST_DEBUG_CATEGORY_INIT (NVDS_APP, "NVDS_APP", 0, NULL); + + if (!g_option_context_parse (ctx, &argc, &argv, &error)) { + NVGSTDS_ERR_MSG_V ("%s", error->message); + g_print ("%s",g_option_context_get_help (ctx, TRUE, NULL)); + return -1; + } + + if (print_version) { + g_print ("deepstream-test5-app version %d.%d.%d\n", + NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO); + return 0; + } + + if (print_dependencies_version) { + g_print ("deepstream-test5-app version %d.%d.%d\n", + NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO); + return 0; + } + + if (cfg_files) { + num_instances = g_strv_length (cfg_files); + } + if (input_files) { + num_input_files = g_strv_length (input_files); + } + + if (!cfg_files || num_instances == 0) { + NVGSTDS_ERR_MSG_V ("Specify config file with -c option"); + return_value = -1; + goto done; + } + + for (i = 0; i < num_instances; i++) { + appCtx[i] = (AppCtx *) g_malloc0 (sizeof (AppCtx)); + appCtx[i]->person_class_id = -1; + appCtx[i]->car_class_id = -1; + appCtx[i]->index = i; + appCtx[i]->active_source_index = -1; + if (show_bbox_text) { + appCtx[i]->show_bbox_text = TRUE; + } + + if (input_files && input_files[i]) { + appCtx[i]->config.multi_source_config[0].uri = + g_strdup_printf ("file://%s", input_files[i]); + g_free (input_files[i]); + } + + if (!parse_config_file (&appCtx[i]->config, cfg_files[i])) { + NVGSTDS_ERR_MSG_V ("Failed to parse config file '%s'", cfg_files[i]); + appCtx[i]->return_value = -1; + goto done; + } + + if (override_cfg_file && override_cfg_file[i]) { + if (!g_file_test (override_cfg_file[i], + G_FILE_TEST_IS_REGULAR | G_FILE_TEST_IS_SYMLINK)) + { + g_print ("Override file %s does not exist, quitting...\n", + override_cfg_file[i]); + appCtx[i]->return_value = -1; + goto done; + } + } + } + + for (i = 0; i < num_instances; i++) { + for (guint j = 0; j < appCtx[i]->config.num_source_sub_bins; j++) { + /** Force the source (applicable only if RTSP) + * to use TCP for RTP/RTCP channels. + * forcing TCP to avoid problems with UDP port usage from within docker- + * container. + * The UDP RTCP channel when run within docker had issues receiving + * RTCP Sender Reports from server + */ + if (force_tcp) + appCtx[i]->config.multi_source_config[j].select_rtp_protocol = 0x04; + } + if (!create_pipeline (appCtx[i], bbox_generated_probe_after_analytics, + NULL, perf_cb, overlay_graphics)) { + NVGSTDS_ERR_MSG_V ("Failed to create pipeline"); + return_value = -1; + goto done; + } + + /* if (appCtx[i]->config.dsanalytics_config.enable){ + GstPad *src_pad = NULL; + GstElement *nvdsanalytics = appCtx[i]->pipeline.common_elements.dsanalytics_bin.elem_dsanalytics; + src_pad = gst_element_get_static_pad (nvdsanalytics, "src"); + if (!src_pad) + g_print ("Unable to get nvdsanalytics src pad\n"); + else + { + gst_pad_add_probe (src_pad, GST_PAD_PROBE_TYPE_BUFFER, + nvdsanalytics_src_pad_buffer_probe, NULL, NULL); + gst_object_unref (src_pad); + } + }*/ + + /** Now add probe to RTPSession plugin src pad */ + for (guint j = 0; j < appCtx[i]->pipeline.multi_src_bin.num_bins; j++) { + testAppCtx->streams[j].id = j; + } + /** In test5 app, as we could have several sources connected + * for a typical IoT use-case, raising the nvstreammux's + * buffer-pool-size to 16 */ + g_object_set (appCtx[i]->pipeline.multi_src_bin.streammux, + "buffer-pool-size", STREAMMUX_BUFFER_POOL_SIZE, NULL); + } + + main_loop = g_main_loop_new (NULL, FALSE); + + _intr_setup (); + g_timeout_add (400, check_for_interrupt, NULL); + + g_mutex_init (&disp_lock); + display = XOpenDisplay (NULL); + for (i = 0; i < num_instances; i++) { + guint j; + + if (!show_bbox_text) { + GstElement *nvosd = appCtx[i]->pipeline.instance_bins[0].osd_bin.nvosd; + g_object_set(G_OBJECT(nvosd), "display-text", FALSE, NULL); + } + + if (gst_element_set_state (appCtx[i]->pipeline.pipeline, + GST_STATE_PAUSED) == GST_STATE_CHANGE_FAILURE) { + NVGSTDS_ERR_MSG_V ("Failed to set pipeline to PAUSED"); + return_value = -1; + goto done; + } + + if (!appCtx[i]->config.tiled_display_config.enable) + continue; + + for (j = 0; j < appCtx[i]->config.num_sink_sub_bins; j++) { + XTextProperty xproperty; + gchar *title; + guint width, height; + XSizeHints hints = {0}; + + if (!GST_IS_VIDEO_OVERLAY (appCtx[i]->pipeline.instance_bins[0].sink_bin. + sub_bins[j].sink)) { + continue; + } + + if (!display) { + NVGSTDS_ERR_MSG_V ("Could not open X Display"); + return_value = -1; + goto done; + } + + if (appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.width) + width = + appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.width; + else + width = appCtx[i]->config.tiled_display_config.width; + + if (appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.height) + height = + appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.height; + else + height = appCtx[i]->config.tiled_display_config.height; + + width = (width) ? width : DEFAULT_X_WINDOW_WIDTH; + height = (height) ? height : DEFAULT_X_WINDOW_HEIGHT; + + hints.flags = PPosition | PSize; + hints.x = appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.offset_x; + hints.y = appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.offset_y; + hints.width = width; + hints.height = height; + + windows[i] = + XCreateSimpleWindow (display, RootWindow (display, + DefaultScreen (display)), hints.x, hints.y, width, height, 2, + 0x00000000, 0x00000000); + + XSetNormalHints(display, windows[i], &hints); + + if (num_instances > 1) + title = g_strdup_printf (APP_TITLE "-%d", i); + else + title = g_strdup (APP_TITLE); + if (XStringListToTextProperty ((char **) &title, 1, &xproperty) != 0) { + XSetWMName (display, windows[i], &xproperty); + XFree (xproperty.value); + } + + XSetWindowAttributes attr = { 0 }; + if ((appCtx[i]->config.tiled_display_config.enable && + appCtx[i]->config.tiled_display_config.rows * + appCtx[i]->config.tiled_display_config.columns == 1) || + (appCtx[i]->config.tiled_display_config.enable == 0 && + appCtx[i]->config.num_source_sub_bins == 1)) { + } else { + attr.event_mask = ButtonPress | KeyRelease; + } + XChangeWindowAttributes (display, windows[i], CWEventMask, &attr); + + Atom wmDeleteMessage = XInternAtom (display, "WM_DELETE_WINDOW", False); + if (wmDeleteMessage != None) { + XSetWMProtocols (display, windows[i], &wmDeleteMessage, 1); + } + XMapRaised (display, windows[i]); + XSync (display, 1); //discard the events for now + gst_video_overlay_set_window_handle (GST_VIDEO_OVERLAY (appCtx + [i]->pipeline.instance_bins[0].sink_bin.sub_bins[j].sink), + (gulong) windows[i]); + gst_video_overlay_expose (GST_VIDEO_OVERLAY (appCtx[i]->pipeline. + instance_bins[0].sink_bin.sub_bins[j].sink)); + if (!x_event_thread) + x_event_thread = g_thread_new ("nvds-window-event-thread", + nvds_x_event_thread, NULL); + } + } + + /* Dont try to set playing state if error is observed */ + if (return_value != -1) { + for (i = 0; i < num_instances; i++) { + if (gst_element_set_state (appCtx[i]->pipeline.pipeline, + GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { + + g_print ("\ncan't set pipeline to playing state.\n"); + return_value = -1; + goto done; + } + } + } + + print_runtime_commands (); + + changemode (1); + + g_timeout_add (40, event_thread_func, NULL); + g_main_loop_run (main_loop); + + changemode (0); + +done: + + g_print ("Quitting\n"); + for (i = 0; i < num_instances; i++) { + if (appCtx[i] == NULL) + continue; + + if (appCtx[i]->return_value == -1) + return_value = -1; + + destroy_pipeline (appCtx[i]); + + g_mutex_lock (&disp_lock); + if (windows[i]) + XDestroyWindow (display, windows[i]); + windows[i] = 0; + g_mutex_unlock (&disp_lock); + + g_free (appCtx[i]); + } + + g_mutex_lock (&disp_lock); + if (display) + XCloseDisplay (display); + display = NULL; + g_mutex_unlock (&disp_lock); + g_mutex_clear (&disp_lock); + + if (main_loop) { + g_main_loop_unref (main_loop); + } + + if (ctx) { + g_option_context_free (ctx); + } + + if (return_value == 0) { + g_print ("App run successful\n"); + } else { + g_print ("App run failed\n"); + } + + gst_deinit (); + + return return_value; + + g_free (testAppCtx); + + return 0; +} + + static gchar * +get_first_result_label (NvDsClassifierMeta * classifierMeta) +{ + GList *n; + for (n = classifierMeta->label_info_list; n != NULL; n = n->next) { + NvDsLabelInfo *labelInfo = (NvDsLabelInfo *) (n->data); + if (labelInfo->result_label[0] != '\0') { + return g_strdup (labelInfo->result_label); + } + } + return NULL; +} + + diff --git a/legacy_apps/deepstream-occupancy-analytics/images/kafka_messages.gif b/legacy_apps/deepstream-occupancy-analytics/images/kafka_messages.gif new file mode 100755 index 0000000..327ea7b Binary files /dev/null and b/legacy_apps/deepstream-occupancy-analytics/images/kafka_messages.gif differ diff --git a/legacy_apps/deepstream-occupancy-analytics/images/test.png b/legacy_apps/deepstream-occupancy-analytics/images/test.png new file mode 100755 index 0000000..9fca462 Binary files /dev/null and b/legacy_apps/deepstream-occupancy-analytics/images/test.png differ diff --git a/legacy_apps/deepstream-occupancy-analytics/includes/analytics.h b/legacy_apps/deepstream-occupancy-analytics/includes/analytics.h new file mode 100755 index 0000000..402f15a --- /dev/null +++ b/legacy_apps/deepstream-occupancy-analytics/includes/analytics.h @@ -0,0 +1,17 @@ +#ifndef _ANALYTICS_H_ +#define _ANALYTICS_H_ + +#include + +/* User defined */ +typedef struct +{ + + guint32 lcc_cnt_exit; + guint32 lccum_cnt; + guint32 lcc_cnt_entry; + guint32 source_id; + +} AnalyticsUserMeta; + +#endif diff --git a/legacy_apps/deepstream-occupancy-analytics/includes/nvdsmeta_schema.h b/legacy_apps/deepstream-occupancy-analytics/includes/nvdsmeta_schema.h new file mode 100755 index 0000000..23f5a02 --- /dev/null +++ b/legacy_apps/deepstream-occupancy-analytics/includes/nvdsmeta_schema.h @@ -0,0 +1,271 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file + * NVIDIA DeepStream: Metadata Extension Structures + * + * @b Description: This file defines the NVIDIA DeepStream metadata structures + * used to describe metadata objects. + */ + +/** + * @defgroup metadata_extensions Metadata Extension Structures + * + * Defines metadata structures used to describe metadata objects. + * + * @ingroup NvDsMetaApi + * @{ + */ + +#ifndef NVDSMETA_H_ +#define NVDSMETA_H_ + +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * Defines event type flags. + */ +typedef enum NvDsEventType { + NVDS_EVENT_ENTRY, + NVDS_EVENT_EXIT, + NVDS_EVENT_MOVING, + NVDS_EVENT_STOPPED, + NVDS_EVENT_EMPTY, + NVDS_EVENT_PARKED, + NVDS_EVENT_RESET, + + /** Reserved for future use. Custom events must be assigned values + greater than this. */ + NVDS_EVENT_RESERVED = 0x100, + /** Specifies a custom event. */ + NVDS_EVENT_CUSTOM = 0x101, + NVDS_EVENT_FORCE32 = 0x7FFFFFFF +} NvDsEventType; + +/** + * Defines object type flags. + */ +typedef enum NvDsObjectType { + NVDS_OBJECT_TYPE_VEHICLE, + NVDS_OBJECT_TYPE_PERSON, + NVDS_OBJECT_TYPE_FACE, + NVDS_OBJECT_TYPE_BAG, + NVDS_OBJECT_TYPE_BICYCLE, + NVDS_OBJECT_TYPE_ROADSIGN, + /** Reserved for future use. Custom objects must be assigned values + greater than this. */ + NVDS_OBJECT_TYPE_RESERVED = 0x100, + /** Specifies a custom object. */ + NVDS_OBJECT_TYPE_CUSTOM = 0x101, + /** "object" key will be missing in the schema */ + NVDS_OBJECT_TYPE_UNKNOWN = 0x102, + NVDS_OBEJCT_TYPE_FORCE32 = 0x7FFFFFFF +} NvDsObjectType; + +/** + * Defines payload type flags. + */ +typedef enum NvDsPayloadType { + NVDS_PAYLOAD_DEEPSTREAM, + NVDS_PAYLOAD_DEEPSTREAM_MINIMAL, + /** Reserved for future use. Custom payloads must be assigned values + greater than this. */ + NVDS_PAYLOAD_RESERVED = 0x100, + /** Specifies a custom payload. You must implement the nvds_msg2p_* + interface. */ + NVDS_PAYLOAD_CUSTOM = 0x101, + NVDS_PAYLOAD_FORCE32 = 0x7FFFFFFF +} NvDsPayloadType; + +/** + * Holds a rectangle's position and size. + */ +typedef struct NvDsRect { + float top; /**< Holds the position of rectangle's top in pixels. */ + float left; /**< Holds the position of rectangle's left side in pixels. */ + float width; /**< Holds the rectangle's width in pixels. */ + float height; /**< Holds the rectangle's height in pixels. */ +} NvDsRect; + +/** + * Holds geolocation parameters. + */ +typedef struct NvDsGeoLocation { + gdouble lat; /**< Holds the location's latitude. */ + gdouble lon; /**< Holds the location's longitude. */ + gdouble alt; /**< Holds the location's altitude. */ +} NvDsGeoLocation; + +/** + * Hold a coordinate's position. + */ +typedef struct NvDsCoordinate { + gdouble x; /**< Holds the coordinate's X position. */ + gdouble y; /**< Holds the coordinate's Y position. */ + gdouble z; /**< Holds the coordinate's Z position. */ +} NvDsCoordinate; + +/** + * Holds an object's signature. + */ +typedef struct NvDsObjectSignature { + /** Holds a pointer to an array of signature values. */ + gdouble *signature; + /** Holds the number of signature values in @a signature. */ + guint size; +} NvDsObjectSignature; + +/** + * Holds a vehicle object's parameters. + */ +typedef struct NvDsVehicleObject { + gchar *type; /**< Holds a pointer to the type of the vehicle. */ + gchar *make; /**< Holds a pointer to the make of the vehicle. */ + gchar *model; /**< Holds a pointer to the model of the vehicle. */ + gchar *color; /**< Holds a pointer to the color of the vehicle. */ + gchar *region; /**< Holds a pointer to the region of the vehicle. */ + gchar *license; /**< Holds a pointer to the license number of the vehicle.*/ +} NvDsVehicleObject; + +/** + * Holds a person object's parameters. + */ +typedef struct NvDsPersonObject { + gchar *gender; /**< Holds a pointer to the person's gender. */ + gchar *hair; /**< Holds a pointer to the person's hair color. */ + gchar *cap; /**< Holds a pointer to the type of cap the person is + wearing, if any. */ + gchar *apparel; /**< Holds a pointer to a description of the person's + apparel. */ + guint age; /**< Holds the person's age. */ +} NvDsPersonObject; + +/** + * Holds a face object's parameters. + */ +typedef struct NvDsFaceObject { + gchar *gender; /**< Holds a pointer to the person's gender. */ + gchar *hair; /**< Holds a pointer to the person's hair color. */ + gchar *cap; /**< Holds a pointer to the type of cap the person + is wearing, if any. */ + gchar *glasses; /**< Holds a pointer to the type of glasses the person + is wearing, if any. */ + gchar *facialhair;/**< Holds a pointer to the person's facial hair color. */ + gchar *name; /**< Holds a pointer to the person's name. */ + gchar *eyecolor; /**< Holds a pointer to the person's eye color. */ + guint age; /**< Holds the person's age. */ +} NvDsFaceObject; + +/** + * Holds event message meta data. + * + * You can attach various types of objects (vehicle, person, face, etc.) + * to an event by setting a pointer to the object in @a extMsg. + * + * Similarly, you can attach a custom object to an event by setting a pointer to the object in @a extMsg. + * A custom object must be handled by the metadata parsing module accordingly. + */ +typedef struct NvDsEventMsgMeta { + /** Holds the event's type. */ + NvDsEventType type; + /** Holds the object's type. */ + NvDsObjectType objType; + /** Holds the object's bounding box. */ + NvDsRect bbox; + /** Holds the object's geolocation. */ + NvDsGeoLocation location; + /** Holds the object's coordinates. */ + NvDsCoordinate coordinate; + /** Holds the object's signature. */ + NvDsObjectSignature objSignature; + /** Holds the object's class ID. */ + gint objClassId; + /** Holds the ID of the sensor that generated the event. */ + gint sensorId; + /** Holds the ID of the analytics module that generated the event. */ + gint moduleId; + /** Holds the ID of the place related to the object. */ + gint placeId; + /** Holds the ID of the component (plugin) that generated this event. */ + gint componentId; + /** Holds the video frame ID of this event. */ + gint frameId; + /** Holds the confidence level of the inference. */ + gdouble confidence; + /** Holds the object's tracking ID. */ + gint trackingId; + /** Holds a pointer to the generated event's timestamp. */ + gchar *ts; + /** Holds a pointer to the detected or inferred object's ID. */ + gchar *objectId; + + /** Holds a pointer to a string containing the sensor's identity. */ + gchar *sensorStr; + /** Holds a pointer to a string containing other attributes associated with + the object. */ + gchar *otherAttrs; + /** Holds a pointer to the name of the video file. */ + gchar *videoPath; + /** Holds a pointer to event message meta data. This can be used to hold + data that can't be accommodated in the existing fields, or an associated + object (representing a vehicle, person, face, etc.). */ + gpointer extMsg; + /** Holds the size of the custom object at @a extMsg. */ + guint extMsgSize; + + /*My data*/ + guint occupancy; + guint source_id; + guint lccum_cnt_entry; + guint lccum_cnt_exit; +} NvDsEventMsgMeta; + +/** + * Holds event information. + */ +typedef struct _NvDsEvent { + /** Holds the type of event. */ + NvDsEventType eventType; + /** Holds a pointer to event metadata. */ + NvDsEventMsgMeta *metadata; +} NvDsEvent; + +/** + * Holds payload metadata. + */ +typedef struct NvDsPayload { + /** Holds a pointer to the payload. */ + gpointer payload; + /** Holds the size of the payload. */ + guint payloadSize; + /** Holds the ID of the component (plugin) which attached the payload + (optional). */ + guint componentId; +} NvDsPayload; + +#ifdef __cplusplus +} +#endif +#endif /* NVDSMETA_H_ */ + +/** @} */ diff --git a/legacy_apps/deepstream-retail-analytics/LICENSE.md b/legacy_apps/deepstream-retail-analytics/LICENSE.md new file mode 100755 index 0000000..06673f9 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/LICENSE.md @@ -0,0 +1,15 @@ +SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/legacy_apps/deepstream-retail-analytics/Makefile b/legacy_apps/deepstream-retail-analytics/Makefile new file mode 100755 index 0000000..cdc241e --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/Makefile @@ -0,0 +1,67 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +APP:= ds-retail-iva + +TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -) + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/bin/ + +ifeq ($(TARGET_DEVICE),aarch64) + CFLAGS:= -DPLATFORM_TEGRA +endif + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes \ + -I /usr/local/cuda-$(CUDA_VER)/include + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) -g + +LIBS:= $(shell pkg-config --libs $(PKGS)) + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lrt \ + -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart -lnvds_yml_parser \ + -lcuda -Wl,-rpath,$(LIB_INSTALL_DIR) + +all: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) + + diff --git a/legacy_apps/deepstream-retail-analytics/README.md b/legacy_apps/deepstream-retail-analytics/README.md new file mode 100755 index 0000000..4e538d9 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/README.md @@ -0,0 +1,185 @@ +# Description + +This is a sample application to perform real-time Intelligent Video Analytics (IVA) in a brick and mortar retail environment using NVIDIA DeepStream, TAO, and pre-trained models. DeepStream is used to run DL inference on a video feed inside a store to detect and track customers, and identify whether the detected persons are carrying shopping baskets. The inference output of this Computer Vision (CV) pipeline is streamed, using Kafka, to a Time-Series Database (TSDB) for archival and further processing. A Django app serves a REST-ful API to query insights based on the inference data. We also demonstrate a sample front-end dashboard to quickly visualize the various Key Performance Indicators (KPIs) available through the Django app. + +This application is based on deepstream-test4 and deepstream-test5 sample applications included with DeepStream. The architecture diagram below shows how all the components are connected. + +![](./media/output.gif) + +What is this DeepStream pipeline made of? + +* Primary Detector: PeopleNet Pre-Trained Model (PTM) from NGC +* Secondary Detector: Custom classification model trained using TAO toolkit to classify people with and without shopping baskets +* Object Tracker: NvDCF tracker +* Message Converter: Custom message converter to generate custom payload from inference data +* Message Broker: Message broker to relay inference data to a kafka server + +# Table of Contents +* [Description](#description) +* [Table of Contents](#table-of-contents) +* [Application Architecture](#application-architecture) +* [Prerequisites](#prerequisites) +* [Getting Started](#getting-started) +* [Build](#build) +* [Running the Application](#run-the-application) +* [Output](#output) +* [Advanced](#advanced) + +# Application Architecture + + +
+ + +# Quick Start + +# Prerequisites + +1. Install the latest [NVIDIA drivers](https://www.nvidia.com/download/index.aspx) for your operating system and GPU. + +2. Install Docker and the NVIDIA Container Toolkit - Refer to this [README](docs/install_nvidia_container_toolkit.md). + +3. **OPTIONAL:** Install python and pip. Required for front-end only. Can omit if not using front-end. + +4. Install DeepStream SDK [instructions](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_Quickstart.html) + + * Pull the docker image for DeepStream development + ```bash + docker pull nvcr.io/nvidia/deepstream:6.1-devel + ``` + + * Allow external applications to connect to the host's X display + ```bash + xhost + + ``` + + **Note:** If you are using a remote machine, the above command will not work from an SSH session. It has to be executed from a VNC/RDP connection. + + * Run the container + ```bash + docker run -it --entrypoint /bin/bash --gpus all --rm --network=host -e DISPLAY=:0 -v /tmp/.X11-unix/:/tmp/.X11-unix --privileged -v /var/run/docker.sock:/var/run/docker.sock nvcr.io/nvidia/deepstream:6.1-devel + ``` + + This command will + * Start the container + * Provide access to all GPUs + * Host the container on the host's network + * Forwards the display of the host to the container along with some other volumes + * Opens an interactive terminal to run commands from within the container + +5. Install git-lfs inside the container + + ```bash + apt install git-lfs + ``` + + +6. We need a kafka message broker and kSQL database. For the purpose of this project, we use [confluent-platform](https://docs.confluent.io/platform/current/quickstart/ce-docker-quickstart.html) to setup these services. So, lets setup confluent-platform: + + **Note:** Bash commands in this section should be run from a separete terminal window and **not from within the DeepStream container**. + + ```bash + wget https://raw.githubusercontent.com/confluentinc/cp-all-in-one/7.2.1-post/cp-all-in-one/docker-compose.yml + docker-compose up -d + ``` + * Verify if all the containers started successfully by running `docker ps` + + ![](./media/docker_container_ps.png) + +7. Create a kafka-topic that will be used to receive messages sent by the DeepStream app + + ```bash + docker exec -it broker /bin/bash + # Within the container + kafka-topics --bootstrap-server "localhost:9092" --topic "detections" --create + ``` + + * You can also create the kafka topic by navigating to the confluent control center > cluster > Topics > Add Topic. + +8. Setup a [kSQL stream](https://docs.ksqldb.io/en/latest/concepts/streams/) based on the topic `detections`: + + a) If you used the above mentioned docker compose file, you can access kSQL CLI by running + ``` + docker exec -it ksqldb-cli ksql http://ksqldb-server:8088 + ``` + b) Once the CLI is active, copy-paste the content from [confluent-platform/stream_creation.sql](confluent-platform/stream_creation.sql) into the CLI to create the stream + + **Note:** You don't have to explicitly create the topic in confluent-kafka. The broker will automatically create a topic once DeepStream sends messages to a new topic. + +# Getting Started + +1. If you are using DeepStream via a docker container as mentioned in the instructions above, execute the following command to open the terminal of the DeepStream docker container if it's not already open. Otherwise, skip this step. + + ```bash + docker exec -it /bin/bash + ``` + + You can locate the container id by running the following command: + + ```bash + docker container ps + ``` + ![](./media/deepstream_container.png) + +2. Clone the repo in $DS_SDK_ROOT/sources/apps/sample_apps/ + ```bash + cd /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps + git clone https://github.com/NVIDIA-AI-IOT/deepstream-retail-analytics.git + cd deepstream-retail-analytics + git lfs pull + ``` + + * Although not necessary, it is recommended to verify the checksum of model and input files to confirm file integrity + ```bash + cd files/ + sha512sum -c checksum.txt + ``` + +3. Download PeopleNet model with the model download script. Download the `.etlt` and `labels.txt` files. + ```bash + bash ./download_models.sh + ``` +4. The custom message converter should be built inside the docker. The build instructions are [custom nvmsgconv library](nvmsgconv/README.md). + +# Build for x86 dGPU system + +Run the following commands from project root + +Modify the below command with the cuda version installed in the docker container. To check the CUDA version inside the docker container you can use `nvcc --version` command. + +```bash +export CUDA_VER= +make -B +``` + +# Run the application + +## Running the DeepStream Application + +```bash +./ds-retail-iva configs/retail_iva.yml --no-display +``` + +The `--no-display` flag in the above command is optional. If the application is running from within a docker container without a display attached, you should use this flag. + +## Running the front-end + +```bash +cd ds-retail-iva-frontend +pip install -r requirements.txt +python3 manage.py runserver 0.0.0.0:8000 +``` + +Open a browser and go to [http://localhost:8000](http://localhost:8000) to visualize the dashboard + +# Output + +**Dashboard** + + + +# Advanced + +[TAO README](./TAO/README.md) - Follow the instructions in this file to create a dataset and train a classification model using TAO toolkit + +[NvMsgConv README](./nvmsgconv/README.md) - Follow this README to build a custom library to modify message payload generated by DeepStream diff --git a/legacy_apps/deepstream-retail-analytics/TAO/README.md b/legacy_apps/deepstream-retail-analytics/TAO/README.md new file mode 100755 index 0000000..b9fdcc8 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/TAO/README.md @@ -0,0 +1,109 @@ +# Training a Classification Model using TAO Toolkit + +# Dataset Structure + +TAO Toolkit requirements + +* One folder for training, validation and testing +* Each of the above folders should contain one folder for each class that the model should learn ("hasBasket" and "noBasket" in this case) + +``` +|--dataset_root: + |--train + |--hasBasket: + |--1.jpg + |--2.jpg + |--noBasket: + |--01.jpg + |--02.jpg + |--val + |--hasBasket: + |--3.jpg + |--4.jpg + |--noBasket: + |--03.jpg + |--04.jpg + |--test + |--hasBasket: + |--5.jpg + |--6.jpg + |--noBasket: + |--05.jpg + |--06.jpg +``` + +If your dataset is in KITTI format (object detection) and you would like to convert it to a classification dataset, you can use the [`kitti_to_classification.py`](kitti_to_classification.py) file provided in this directory. + + +# Training a classification model + +## Prerequisites + +* Installation of TAO toolkit +* Download a PTM (pre-trained model) from NGC. We will use [resnet34](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/pretrained_detectnet_v2/files) for this example + +**Note:** You could use NGC command line tool or curl to download the model + +## Steps to train a classification model + +Refer to [TAO documentation](https://docs.nvidia.com/tao/tao-toolkit/text/image_classification.html) for more information on how to modify the spec file for training. + +We need to set a key to encode the model. The same key is to be used when saving/loading the model. In this example the key is set as `nvidia_tlt` + +* Create a directory to store checkpoints + +```bash +mkdir classification_model +``` + +* Train the model + +```bash +tao classification train -e SPECS_classification_train.txt -k nvidia_tlt -r classification_model +``` + +* Evaluate the model + +The path for the evaluation dataset is specified in the `eval_config` in the spec file + +```bash +tao classification evaluate -e SPECS_classification_train.txt -k nvidia_tlt +``` + +* OPTIONAL: Running inference on an image/directory using the model + +```bash +tao classification inference -m -i -k nvidia_tlt -cm -e SPECS_classification_train.txt +``` + +* Exporting the model + +Checkpoints for the model trained are located in the `classification_model/weights` directory as `.tlt` files(The directory to store output is set during the training step). + +Training logs are located in the same folder as JSON and CSV files. + +Pick a model to export depending on the loss/accuracy values + +```bash +tao classification export -m -k nvidia_tlt -o basketClassifier.etlt +``` + +* Deploying the model using DeepStream + +There are two ways to use the above exported model with DeepStream + +**Option 1:** Use the above exported `.etlt` model directly with DeepStream. + +**Option 2:** Use the `tao-converter` and generate a device specific engine file. The generated engine file should also be specified in the config file for the inference engine. Refer to [basket_classifier.yml](../configs/basket_classifier.yml) for an example config file. + +**Generating an engine using tao-converter** + +*Output Nodes:* Since this is a classification model, there is only only one classification node "predictions/Softmax" + +*Dimensions:* Dimensions for the below command should be the same as it was specified in the [SPECS_classification_train.txt](./SPECS_classification_train.txt) + +*input_file:* The input for the below command is the model exported in the previous command + +```bash +tao-converter -k nvidia_tlt -d 3,224,224 -o predictions/Softmax basketClassifier.etlt +``` \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/TAO/SPECS_classification_train.txt b/legacy_apps/deepstream-retail-analytics/TAO/SPECS_classification_train.txt new file mode 100755 index 0000000..d7bfdf6 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/TAO/SPECS_classification_train.txt @@ -0,0 +1,79 @@ +model_config { + # Model Architecture can be chosen from: + # ['resnet', 'vgg', 'googlenet', 'alexnet'] + arch: "resnet" + # for resnet --> n_layers can be [10, 18, 50] + # for vgg --> n_layers can be [16, 19] + n_layers: 34 + use_batch_norm: True + use_bias: False + all_projections: False + use_pooling: True + retain_head: True + resize_interpolation_method: BICUBIC + # if you want to use the pretrained model, + # image size should be "3,224,224" + # otherwise, it can be "3, X, Y", where X,Y >= 16 + input_image_size: "3,224,224" +} +train_config { + val_dataset_path: "/work/basketDetection/classification_data/" + train_dataset_path: "/work/basketDetection/classification_data/" + pretrained_model_path: "/work/basketDetection/resnet_34.hdf5" + # Only ['sgd', 'adam'] are supported for optimizer + optimizer { + sgd { + lr: 0.01 + decay: 0.0 + momentum: 0.9 + nesterov: False + } + } + batch_size_per_gpu: 50 + n_epochs: 150 + # Number of CPU cores for loading data + n_workers: 16 + # regularizer + reg_config { + # regularizer type can be "L1", "L2" or "None". + type: "L2" + # if the type is not "None", + # scope can be either "Conv2D" or "Dense" or both. + scope: "Conv2D,Dense" + # 0 < weight decay < 1 + weight_decay: 0.000015 + } + # learning_rate + lr_config { + cosine { + learning_rate: 0.04 + soft_start: 0.0 + } + } + enable_random_crop: False + enable_center_crop: False + enable_color_augmentation: True + mixup_alpha: 0.2 + label_smoothing: 0.1 + preprocess_mode: "caffe" + image_mean { + key: 'b' + value: 103.9 + } + image_mean { + key: 'g' + value: 116.8 + } + image_mean { + key: 'r' + value: 123.7 + } +} +eval_config { + eval_dataset_path: "/work/basketDetection/classification_data/" + model_path: "/work/basketDetection/classification_model/weights/resnet_150.tlt" + top_k: 3 + batch_size: 256 + n_workers: 8 + enable_center_crop: False +} \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/TAO/kitti_to_classification.py b/legacy_apps/deepstream-retail-analytics/TAO/kitti_to_classification.py new file mode 100755 index 0000000..7e4c22c --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/TAO/kitti_to_classification.py @@ -0,0 +1,46 @@ +import os +from glob import glob +import cv2 +from tqdm import tqdm + + +def read_label_file(filepath): + labels = [] + coordinates = [] + + # Read the file + with open(filepath, "r") as file: + lines = file.readlines() + + for line in lines: + line = line.split() + lab, _, _, _, x1, y1, x2, y2, _, _, _, _, _, _, _ = line + x1, y1, x2, y2 = map(float, [x1, y1, x2, y2]) + x1, y1, x2, y2 = map(int, [x1, y1, x2, y2]) + labels.append(lab) + coordinates.append((x1, y1, x2, y2)) + + return labels, coordinates + + +def crop_image(img, xmin, ymin, xmax, ymax): + return img[ymin:ymax, xmin:xmax :] + + +def draw_bbox(img, xmin, ymin, xmax, ymax): + img = cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 0, 0), 3) + return img + + +if __name__ == "__main__": + images = glob("default/image_2/*.PNG") + + for image in tqdm(images): + label_file, _ = os.path.splitext(os.path.basename(image)) + labels, coordinates = read_label_file(f"default/label_2/{label_file}.txt") + img = cv2.imread(image) + for i, label in enumerate(labels): + img_cropped = crop_image(img, coordinates[i][0], coordinates[i][1], coordinates[i][2], coordinates[i][3]) + # img_annotated = draw_bbox(img, coordinates[i][0], coordinates[i][1], coordinates[i][2], coordinates[i][3]) + cv2.imwrite(f"classification_data/{label}/{label_file}_{i}.png", img_cropped) + # cv2.imwrite(f"test_{i}.png", img_annotated) \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/THIRD_PARTY_LICENSE b/legacy_apps/deepstream-retail-analytics/THIRD_PARTY_LICENSE new file mode 100755 index 0000000..1186e01 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/THIRD_PARTY_LICENSE @@ -0,0 +1,5 @@ +python-ceriti: https://github.com/certifi/python-certifi/blob/master/LICENSE + +This Source Code Form is subject to the terms of the Mozilla Public License, +v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain +one at http://mozilla.org/MPL/2.0/. diff --git a/legacy_apps/deepstream-retail-analytics/configs/README.md b/legacy_apps/deepstream-retail-analytics/configs/README.md new file mode 100755 index 0000000..0950c67 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/configs/README.md @@ -0,0 +1,16 @@ +# Config files for the Project + +README.md + +`retail_iva.yml`: Config file for this app. Holds information about input parameters for various plugins + +`pgie_config_peoplenet.yml`: Config file ofr Primary Inference Engine (PeopleNet) + +`basket_classifier.yml`, `basket_classifier.txt`: Config file for Secondary Inference Engine + +`dstest4_msgconv_config.txt`, `dstest4_msgconv_config.yml`: Default config file for message converter + +`dstest4_tracker_config.txt`: Config file for NvDCF tracker + +`dstest4_msgconv_config_1.txt`, `dstest4_msgconv_config_1.yml`: Config file we use in this application for message converter + diff --git a/legacy_apps/deepstream-retail-analytics/configs/basket_classifier.txt b/legacy_apps/deepstream-retail-analytics/configs/basket_classifier.txt new file mode 100755 index 0000000..c424eb9 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/configs/basket_classifier.txt @@ -0,0 +1,44 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[property] +gpu-id=0 +# preprocessing parameters=These are the same for all classification models generated by TAO Toolkit. +net-scale-factor=1.0 +offsets=103.939;116.779;123.68 +model-color-format=1 + +# Model specific paths. These need to be updated for every classification model. +labelfile-path=/work/basketDetection/classification_model/labels.txt +tlt-encoded-model=/work/basketDetection/basketClassifier.etlt +model-engine-file=/work/basketDetection/basketClassifier.etlt_b1_gpu0_fp32.engine +tlt-model-key=nvidia_tlt +infer-dims=3;224;224 # where c = number of channels, h = height of the model input, w = width of model input +uff-input-blob-name=input_1 +output-blob-names=predictions/Softmax +operate-on-gie-id=1 +# operate-on-class-ids=1 +batch-size=1 + +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=0 +# process-mode=2 - inferences on crops from primary detector, 1 - inferences on whole frame +process-mode=2 +interval=0 +network-type=1 # defines that the model is a classifier. +gie-unique-id=2 +classifier-threshold=0.2 diff --git a/legacy_apps/deepstream-retail-analytics/configs/basket_classifier.yml b/legacy_apps/deepstream-retail-analytics/configs/basket_classifier.yml new file mode 100755 index 0000000..64f20b2 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/configs/basket_classifier.yml @@ -0,0 +1,45 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +property: + gpu-id: 0 + # preprocessing parameters: These are the same for all classification models generated by TAO Toolkit. + net-scale-factor: 1.0 + offsets: 103.939;116.779;123.68 + model-color-format: 1 + + # Model specific paths. These need to be updated for every classification model. + labelfile-path: ../files/basket_classifier_labels.txt + tlt-encoded-model: ../files/basketClassifier.etlt + # model-engine-file: ../files/basketClassifier.etlt_b1_gpu0_fp32.engine + tlt-model-key: nvidia_tlt + infer-dims: 3;224;224 # where c = number of channels, h = height of the model input, w = width of model input + uff-input-blob-name: input_1 + output-blob-names: predictions/Softmax + # operate-on-gie-id: 1 + # operate-on-class-ids: 1 + batch-size: 1 + classifier-type: hasBasketClassifier + + ## 0=FP32, 1=INT8, 2=FP16 mode + network-mode: 0 + # process-mode: 2 - inferences on crops from primary detector, 1 - inferences on whole frame + process-mode: 2 + interval: 0 + network-type: 1 # defines that the model is a classifier. + gie-unique-id: 2 + classifier-threshold: 0.5 diff --git a/legacy_apps/deepstream-retail-analytics/configs/dstest4_config.yml b/legacy_apps/deepstream-retail-analytics/configs/dstest4_config.yml new file mode 100755 index 0000000..f2d30b1 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/configs/dstest4_config.yml @@ -0,0 +1,53 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +source: + # location: ../../../../samples/streams/sample_720p.h264 + location: compressed.h264 + +streammux: + batch-size: 1 + batched-push-timeout: 40000 + width: 1280 + height: 720 + +tracker: + tracker-width: 640 + tracker-height: 384 + gpu-id: 0 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_IOU.yml + ll-config-file: ../../../../samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml + # ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_DeepSORT.yml + enable-batch-process: 1 + +msgconv: + msg2p-lib: /opt/nvidia/deepstream/deepstream-6.1/sources/libs/nvmsgconv/libnvds_msgconv.so + payload-type: 0 + msg2p-newapi: 0 + frame-interval: 30 + +msgbroker: + proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + conn-str: localhost;9092 + topic: detections + sync: 0 + +sink: + sync: 1 diff --git a/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config.txt b/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config.txt new file mode 100755 index 0000000..429890e --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config.txt @@ -0,0 +1,53 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[sensor0] +enable=1 +type=Camera +id=CAMERA_ID +location=45.293701447;-75.8303914499;48.1557479338 +description="Entrance of Garage Right Lane" +coordinate=5.2;10.1;11.2 + +[place0] +enable=1 +id=1 +type=garage +name=XYZ +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=walsh +place-sub-field2=lane1 +place-sub-field3=P2 + +[place1] +enable=1 +id=1 +type=garage +name=XYZ +location=28.47;47.46;1.53 +coordinate=1.0;2.0;3.0 +place-sub-field1="C-76-2" +place-sub-field2="LEV/EV/CP/ADA" +place-sub-field3=P2 + +[analytics0] +enable=1 +id=XYZ +description="Vehicle Detection and License Plate Recognition" +source=OpenALR +version=1.0 \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config.yml b/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config.yml new file mode 100755 index 0000000..0025a85 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config.yml @@ -0,0 +1,53 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +sensor0: + enable: 1 + type: Camera + id: CAMERA_ID + location: 45.293701447;-75.8303914499;48.1557479338 + description: "Entrance of Garage Right Lane" + coordinate: 5.2;10.1;11.2 + +place0: + enable: 1 + id: 1 + type: garage + name: XYZ + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: walsh + place-sub-field2: lane1 + place-sub-field3: P2 + +place1: + enable: 1 + id: 1 + type: garage + name: XYZ + location: 28.47;47.46;1.53 + coordinate: 1.0;2.0;3.0 + place-sub-field1: "C-76-2" + place-sub-field2: "LEV/EV/CP/ADA" + place-sub-field3: P2 + +analytics0: + enable: 1 + id: XYZ + description: "Vehicle Detection and License Plate Recognition" + source: OpenALR + version: 1.0 \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config_1.txt b/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config_1.txt new file mode 100755 index 0000000..e014424 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config_1.txt @@ -0,0 +1,16 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ diff --git a/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config_1.yml b/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config_1.yml new file mode 100755 index 0000000..5928c51 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config_1.yml @@ -0,0 +1,16 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ diff --git a/legacy_apps/deepstream-retail-analytics/configs/dstest4_tracker_config.txt b/legacy_apps/deepstream-retail-analytics/configs/dstest4_tracker_config.txt new file mode 100755 index 0000000..aec800c --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/configs/dstest4_tracker_config.txt @@ -0,0 +1,34 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Mandatory properties for the tracker: +# tracker-width, tracker-height: needs to be multiple of 32 for NvDCF and DeepSORT +# gpu-id +# ll-lib-file: path to low-level tracker lib +# ll-config-file: required to set different tracker types +# +[tracker] +tracker-width=960 +tracker-height=544 +gpu-id=0 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +# ll-config-file required to set different tracker types +# ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_IOU.yml +# ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml +ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml +# ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_DeepSORT.yml +enable-batch-process=1 diff --git a/legacy_apps/deepstream-retail-analytics/configs/pgie_config_peoplenet.yml b/legacy_apps/deepstream-retail-analytics/configs/pgie_config_peoplenet.yml new file mode 100755 index 0000000..df687c4 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/configs/pgie_config_peoplenet.yml @@ -0,0 +1,77 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 0.0039215697906911373 + infer-dims: 3;544;960 + onnx-file: ../models/peoplenet/resnet34_peoplenet_int8.onnx + int8-calib-file: ../models/peoplenet/resnet34_peoplenet_int8.txt + labelfile-path: ../models/peoplenet/labels.txt + model-engine-file: ../models/peoplenet/resnet34_peoplenet_int8.onnx_b1_gpu0_int8.engine + batch-size: 1 + process-mode: 1 + model-color-format: 0 + # 0: FP32, 1: INT8, 2: FP16 mode + network-mode: 0 + num-detected-classes: 3 + interval: 0 + gie-unique-id: 1 + output-blob-names: output_bbox/BiasAdd:0;output_cov/Sigmoid:0 + cluster-mode: 3 + # We are interested only in the person class. Filter out bag and face classes. + filter-out-class-ids: 1;2 # Filter out bag and face class. + +# Use the config params below for NMS clustering mode +class-attrs-all: + topk: 20 + nms-iou-threshold: 0.5 + pre-cluster-threshold: 0.3 + minBoxes: 3 + dbscan-min-score: 1.3 + eps: 0.15 + detected-min-w: 20 + detected-min-h: 20 + threshold: 0.7 + +# [property] +# ## model-specific params like paths to model, engine, label files, etc. are to be added by users + +# gpu-id=0 +# net-scale-factor=0.0039215697906911373 +# input-dims=3;544;960;0 +# uff-input-blob-name=input_1 +# process-mode=1 +# model-color-format=0 +# ## 0=FP32, 1=INT8, 2=FP16 mode +# network-mode=1 +# num-detected-classes=3 +# interval=0 +# gie-unique-id=1 +# output-blob-names=output_cov/Sigmoid;output_bbox/BiasAdd +# ## 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +# cluster-mode=3 +# maintain-aspect-ratio=1 + +# [class-attrs-all] +# pre-cluster-threshold=0.3 +# nms-iou-threshold=0.5 +# minBoxes=3 +# dbscan-min-score=1.3 +# eps=0.15 +# detected-min-w=20 +# detected-min-h=20 diff --git a/legacy_apps/deepstream-retail-analytics/configs/retail_iva.yml b/legacy_apps/deepstream-retail-analytics/configs/retail_iva.yml new file mode 100755 index 0000000..3ebe10b --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/configs/retail_iva.yml @@ -0,0 +1,62 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +source: + # location: ../../../../samples/streams/sample_720p.h264 + # location: /work/retail_cropped.264 + # location: /work/data/trimmed.h264 + location: ../deepstream-retail-analytics/files/0001_compressed.h264 + # location: rtsp:// + +streammux: + batch-size: 1 + batched-push-timeout: 40000 + width: 1920 + height: 1080 + + +# tracker: +# tracker-width: 640 +# tracker-height: 384 +# gpu-id: 0 +# ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +# # ll-config-file required to set different tracker types +# # ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_IOU.yml +# # ll-config-file: ../../../../samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml +# ll-config-file: ../../../../samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml +# # ll-config-file=../../../../samples/configs/deepstream-app/config_tracker_DeepSORT.yml +# enable-batch-process: 1 + +msgconv: + msg2p-lib: ../nvmsgconv/libnvds_msgconv.so + #msg2p-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv.so + payload-type: 0 + msg2p-newapi: 0 + frame-interval: 30 + +msgbroker: + proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + conn-str: localhost;9092 + topic: detections + sync: 0 + +sink: + sync: 1 + +filesink: + location: /opt/nvidia/deepstream/deepstream-6.1/sources/apps/sample_apps/deepstream-retail-analytics/retail_output.mp4 + sync: 0 diff --git a/legacy_apps/deepstream-retail-analytics/confluent-platform/docker-compose.yml b/legacy_apps/deepstream-retail-analytics/confluent-platform/docker-compose.yml new file mode 100755 index 0000000..5d5e6f0 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/confluent-platform/docker-compose.yml @@ -0,0 +1,177 @@ +--- +version: '2' +services: + zookeeper: + image: confluentinc/cp-zookeeper:7.1.1 + hostname: zookeeper + container_name: zookeeper + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + broker: + image: confluentinc/cp-server:7.1.1 + hostname: broker + container_name: broker + depends_on: + - zookeeper + ports: + - "9092:9092" + - "9101:9101" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 + KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_JMX_PORT: 9101 + KAFKA_JMX_HOSTNAME: localhost + KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081 + CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092 + CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 + CONFLUENT_METRICS_ENABLE: 'true' + CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' + + schema-registry: + image: confluentinc/cp-schema-registry:7.1.1 + hostname: schema-registry + container_name: schema-registry + depends_on: + - broker + ports: + - "8081:8081" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + + connect: + image: cnfldemos/cp-server-connect-datagen:0.5.3-7.1.0 + hostname: connect + container_name: connect + depends_on: + - broker + - schema-registry + ports: + - "8083:8083" + environment: + CONNECT_BOOTSTRAP_SERVERS: 'broker:29092' + CONNECT_REST_ADVERTISED_HOST_NAME: connect + CONNECT_GROUP_ID: compose-connect-group + CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 + CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter + CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 + # CLASSPATH required due to CC-2422 + CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.1.1.jar + CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" + CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" + CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" + CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR + + control-center: + image: confluentinc/cp-enterprise-control-center:7.1.1 + hostname: control-center + container_name: control-center + depends_on: + - broker + - schema-registry + - connect + - ksqldb-server + ports: + - "9021:9021" + environment: + CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092' + CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'connect:8083' + CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://ksqldb-server:8088" + CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: "http://localhost:8088" + CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" + CONTROL_CENTER_REPLICATION_FACTOR: 1 + CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1 + CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1 + CONFLUENT_METRICS_TOPIC_REPLICATION: 1 + PORT: 9021 + + ksqldb-server: + image: confluentinc/cp-ksqldb-server:7.1.1 + hostname: ksqldb-server + container_name: ksqldb-server + depends_on: + - broker + - connect + ports: + - "8088:8088" + environment: + KSQL_CONFIG_DIR: "/etc/ksql" + KSQL_BOOTSTRAP_SERVERS: "broker:29092" + KSQL_HOST_NAME: ksqldb-server + KSQL_LISTENERS: "http://0.0.0.0:8088" + KSQL_CACHE_MAX_BYTES_BUFFERING: 0 + KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" + KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" + KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" + KSQL_KSQL_CONNECT_URL: "http://connect:8083" + KSQL_KSQL_LOGGING_PROCESSING_TOPIC_REPLICATION_FACTOR: 1 + KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: 'true' + KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: 'true' + + ksqldb-cli: + image: confluentinc/cp-ksqldb-cli:7.1.1 + container_name: ksqldb-cli + depends_on: + - broker + - connect + - ksqldb-server + entrypoint: /bin/sh + tty: true + + ksql-datagen: + image: confluentinc/ksqldb-examples:7.1.1 + hostname: ksql-datagen + container_name: ksql-datagen + depends_on: + - ksqldb-server + - broker + - schema-registry + - connect + command: "bash -c 'echo Waiting for Kafka to be ready... && \ + cub kafka-ready -b broker:29092 1 40 && \ + echo Waiting for Confluent Schema Registry to be ready... && \ + cub sr-ready schema-registry 8081 40 && \ + echo Waiting a few seconds for topic creation to finish... && \ + sleep 11 && \ + tail -f /dev/null'" + environment: + KSQL_CONFIG_DIR: "/etc/ksql" + STREAMS_BOOTSTRAP_SERVERS: broker:29092 + STREAMS_SCHEMA_REGISTRY_HOST: schema-registry + STREAMS_SCHEMA_REGISTRY_PORT: 8081 + + rest-proxy: + image: confluentinc/cp-kafka-rest:7.1.1 + depends_on: + - broker + - schema-registry + ports: + - 8082:8082 + hostname: rest-proxy + container_name: rest-proxy + environment: + KAFKA_REST_HOST_NAME: rest-proxy + KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092' + KAFKA_REST_LISTENERS: "http://0.0.0.0:8082" + KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081' diff --git a/legacy_apps/deepstream-retail-analytics/confluent-platform/stream_creation.sql b/legacy_apps/deepstream-retail-analytics/confluent-platform/stream_creation.sql new file mode 100755 index 0000000..478cc3a --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/confluent-platform/stream_creation.sql @@ -0,0 +1,26 @@ +CREATE STREAM DETECTIONS_STREAM ( + messageid varchar, + mdsversion varchar, + timestamp varchar, + object struct< + id varchar, + speed int, + direction int, + orientation int, + detection varchar, + obj_prop struct< + hasBasket varchar, + confidence double>, + bbox struct< + topleftx int, + toplefty int, + bottomrightx int, + bottomrighty int>>, + event_des struct< + id varchar, + type varchar>, + videopath varchar) WITH ( + KAFKA_TOPIC='detections', + VALUE_FORMAT='JSON', + TIMESTAMP='timestamp', + TIMESTAMP_FORMAT='yyyy-MM-dd''T''HH:mm:ss.SSS''Z'''); \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/docs/install_nvidia_container_toolkit.md b/legacy_apps/deepstream-retail-analytics/docs/install_nvidia_container_toolkit.md new file mode 100755 index 0000000..0224374 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/docs/install_nvidia_container_toolkit.md @@ -0,0 +1,50 @@ + +# NVIDIA Container Toolkit + +* To install NVIDIA container toolkit follow these instructions: + + 1. Setup docker repository + ```bash + sudo apt-get update + + sudo apt-get install ca-certificates curl gnupg lsb-release + + sudo mkdir -p /etc/apt/keyrings + + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + ``` + + 2. Install [Docker](https://docs.docker.com/engine/install/) + ```bash + sudo apt-get update + + sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin + ``` + + 3. Install [nvidia-docker](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker) + ```bash + distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \ + && curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \ + && curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | \ + sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \ + sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list + ``` + + 4. **OPTIONAL:** Post-install instructions to run docker without sudo + + ```bash + sudo groupadd docker + sudo usermod -aG docker $USER + ``` + + ```bash + sudo apt-get update + sudo apt-get install -y nvidia-docker2 + ``` + 5. Restart docker + + ```bash + sudo systemctl restart docker + ``` \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/download_models.sh b/legacy_apps/deepstream-retail-analytics/download_models.sh new file mode 100755 index 0000000..87fa3c1 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/download_models.sh @@ -0,0 +1,24 @@ +#!/bin/sh +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +mkdir -p ./models/peoplenet +cd ./models/peoplenet +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.3/files?redirect=true&path=resnet34_peoplenet_int8.onnx' -O resnet34_peoplenet_int8.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.3/files?redirect=true&path=resnet34_peoplenet_int8.txt' -O resnet34_peoplenet_int8.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.3/files?redirect=true&path=labels.txt' -O labels.txt + diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/README.md b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/README.md new file mode 100755 index 0000000..9502d6a --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/README.md @@ -0,0 +1,165 @@ +# DeepStream Retail IVA Frontend + +This repo is a django-based frontend application for [DeepStream Retail IVA](https://gitlab-master.nvidia.com/admantri/ds-retail-iva). This application will be referred to as the parent application henceforth. + +# Prerequisites + +* Working installation of python and pip. Preferred to use conda + + * Installing anaconda/miniconda + + ```bash + wget https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh + chmod +x Miniconda3-py37_4.12.0-Linux-x86_64.sh + ./Miniconda3-py37_4.12.0-Linux-x86_64.sh + ``` + + * You can create a conda environment using the [environment.yml](./environment.yml) file provided in this repo + + ```bash + conda env create -f environment.yml + ``` + + * If you already have python installed and prefer to not use conda, you can setup the required packages by using the [requirements.txt](./requirements.txt) file + + ```bash + pip install -r requirements.txt + ``` + +* This project assumes that you have already setup confluent platform with kSQL database running as instructed in the [prerequisites](https://gitlab-master.nvidia.com/admantri/ds-retail-iva#prerequisites) section of the parent application. + + +# Running the dashboard + +```bash +python manage.py runserver 0.0.0.0:8000 +``` + +# URLs supported by this API + +All the endpoints supported by this application can be found in the [urls.py](./retail_iva/urls.py) file of the project. + +* http://localhost:8000/ - Homepage where a dashboard with all the plots is shown +* http://localhost:8000/num-visitors-region/ - API endpoint to get the number of visitors in a given region. + + Inputs for this endpoint + + * topleftx - x coordinate of the top left corner + * toplefty - y coordinate of the top left corner + * bottomrightx - x coordinate of the bottom right corner + * bottomrighty - y coordinate of the bottom right corner + + Example URL: http://localhost:8000/num-visitors-region?topleftx=50&bottomrightx=1400&toplefty=0&bottomrighty=1440 + +* http://localhost:8000/num-visitors-time/ - API endpoint to get the number of visitors in a given time window + + Inputs for this endpoint + + * start_time: Starting time of the time window + * end_time: Ending time of the time window + + The API returns 0 if start_time > end_time + + This API supports three forms of inputs + + * If both start_time and end_time are provided, the API calculates the number + of visitors that arrived in that time window + * If only start_time is provided, the API calculates the number of visitors + that arrived after that time + * If only end_time is provided, the API calculates the number of visitors + that arrived before that time + * If none are provided, the API returns "-1" + + Example URL: http://localhost:8000/num-visitors-time?start_time=2022-07-29T03:44:41&end_time=2022-07-29T03:44:45 + +* http://localhost:8000/visitor-path - API endpoint to get the path of a visitor in the store. This API renders a path of the visitor on the background of the store. + + Inputs for this endpoint + + * person_id: This is the ID given by the NvDCF tracker + + Example URL: http://localhost:8000/visitor-path?person_id=7 + + +# Config Files + +This project comes with two config files + +## [store_config.ini](./store_config.ini) + +This is a config file that carries information about coordinates of each aisle. This config file is used to generate a bar chart shown on the dashboard with information about how many visitors are present in each aisle. Adding new sections to this file will automatically result in a refreshed bar graph. + +Each section in the bar graph requires 4 coordinates: + +* topleftx - x coordinate of the top left corner +* toplefty - y coordinate of the top left corner +* bottomrightx - x coordinate of the bottom right corner +* bottomrighty - y coordinate of the bottom right corner + + +## [config.py](./config.py) + +This is the configuration file for the Django project. + +`TEST_MODE` - Set this to True to show a dashboard with fake data generated by running [random_message_generator.py](./random_message_generator.py) + +`ksql_server` - URL for the kSQL server. + +`kafka_server` - URL for the kafka server + +`ksql_stream_name` and `kafka_topic` are set depending on `TEST_MODE`. + + +# Generating random data for testing + +* [`random_message_generator.py`](./random_message_generator.py) can be used to generate fake kafka message data and deliver it to the kafka server. + +* Although, kafka server creates the topic as messages are delivered, we have to explicitly create a kSQL stream. It is important to create the stream since the dashboard is entirely dependant on the kSQL stream. Follow the instructions given in the parent application's README to create a stream. + +```SQL +CREATE STREAM TEST_STREAM ( +messageid varchar, +mdsversion varchar, +timestamp varchar, +object struct< + id varchar, + speed int, + direction int, + orientation int, + detection varchar, + obj_prop struct< + hasBasket varchar, + confidence double>, + bbox struct< + topleftx int, + toplefty int, + bottomrightx int, + bottomrighty int>>, +event_des struct< + id varchar, + type varchar>, +videopath varchar) WITH ( + KAFKA_TOPIC='detections', + VALUE_FORMAT='JSON', + TIMESTAMP='timestamp', + TIMESTAMP_FORMAT='yyyy-MM-dd''T''HH:mm:ss.SSS''Z'''); + +``` + +* To check if the stream was created successfully run `list streams` from the ksql-cli + +* Modify [this line](./random_message_generator.py#L69) to change the number of messages generated. + + +* To generate fake data and send messages execute + +```bash +python random_message_generator.py +``` + +* To verify if the messages were stored in the stream, run the below command from ksql-cli + +```SQL +select * from test_stream +``` + diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/admin.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/admin.py new file mode 100755 index 0000000..8c38f3f --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/apps.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/apps.py new file mode 100755 index 0000000..258d4dd --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class AnalyticsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'analytics' diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/influxdb_connecter.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/influxdb_connecter.py new file mode 100755 index 0000000..1c23edd --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/influxdb_connecter.py @@ -0,0 +1,41 @@ +import sys +import cv2 +import numpy as np +from influxdb import InfluxDBClient as idb +from PIL import Image + +def get_num_visitors_time_window(start_time, end_time, client): + pass + +def get_num_visitors_in_region(topleftx, bottomrightx, toplefty, bottomrighty, client): + client.switch_database("detections") + result = client.query(f'SELECT * FROM detections where \ + topleftx > {topleftx} and bottomrightx < {bottomrightx} and \ + toplefty > {toplefty} and bottomrighty < {bottomrighty}') + + num_results = len(result.raw["series"][0]["values"]) + + return num_results, result + +def get_visitor_path(person_id, client): + client.switch_database("detections") + result = client.query(f'SELECT * FROM detections where id=\'{person_id}\'') + + path = [] + + for val in result.raw["series"][0]["values"]: + bottomrightx = val[1] + bottomrighty = val[2] + topleftx = val[5] + toplefty = val[6] + path.append([(bottomrightx + topleftx)//2, (bottomrighty + toplefty)//2]) + + # img = cv2.imread("frame0.jpg") + img = np.zeros([1080,1920,3],dtype=np.uint8) + img.fill(0) + path = np.array(path) + cv2.drawContours(img, [path], 0, (255,255,255), 2) + cv2.imwrite("testImg.png", img) + + return path, Image.fromarray(img) + diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/ksqldb_connecter.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/ksqldb_connecter.py new file mode 100755 index 0000000..8d0720f --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/ksqldb_connecter.py @@ -0,0 +1,281 @@ +import sys +sys.path.append("../retail_iva") + +import ast +import datetime +from configparser import ConfigParser +import cv2 +import numpy as np +import pandas as pd +from PIL import Image +from ksql import KSQLAPI +import matplotlib.pyplot as plt +# import imageio +from tqdm import tqdm +import config + + +def query_parser(query): + """ + Utility function to parse the query generator object returned by KSQLAPI + This function returns a list of all rows of the table as retuened by the query + """ + # The first element returned by the generator object is a list containing the column headers + # Skip the first row + next(query) + + # Iterate through the rest of the output + output = [item[:-2] for item in query] + + # The last element returned by the generator object is always "]" + # Remove the last element + output = output[:-1] + return output + + +def get_num_visitors_time_window(client, start_time=None, end_time=None): + """ + This API can perform the following functions based on the inputs provided + + * If both start_time and end_time are provided, the API calculates the number + of visitors that arrived in that time window + * If only start_time is provided, the API calculates the number of visitors + that arrived after that time + * If only end_time is provided, the API calculates the number of visitors + that arrived before that time + """ + + # Both start_time and end_time are not None + if start_time is not None and end_time is not None: + query = client.query(f'select object->id from {config.ksql_stream_name} where \ + rowtime > \'{start_time}\' and rowtime < \'{end_time}\'') + # Only start_time is provided + elif start_time is not None and end_time is None: + query = client.query(f'select object->id from {config.ksql_stream_name} where \ + rowtime > \'{start_time}\'') + # Only end_time is provided + elif start_time is None and end_time is not None: + query = client.query(f'select object->id from {config.ksql_stream_name} where \ + rowtime < \'{end_time}\'') + else: + return -1 + + # Parse the query + output = query_parser(query) + + # Count the number of unique objects and return it + # It is important to convert it to a set to avoid duplicates in counting + items = [item for item in output] + items = set(items) + + return len(items), items + + +def get_num_visitors_in_region(topleftx, bottomrightx, toplefty, bottomrighty, client): + """ + This API is used to fetch the number of visitors in a rectangle defined by its + top left and bottom right corners + """ + topleftx, toplefty = int(topleftx), int(toplefty) + bottomrightx, bottomrighty = int(bottomrightx), int(bottomrighty) + img = cv2.imread("background.png") + cv2.rectangle(img, (topleftx, toplefty), (bottomrightx, bottomrighty), (0, 0, 255), 3) + cv2.imwrite("roi.png", img) + + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + query = client.query(f'select object->id from {config.ksql_stream_name} where \ + object->bbox->topleftx > {topleftx} and \ + object->bbox->bottomrightx < {bottomrightx} and \ + object->bbox->toplefty > {toplefty} and \ + object->bbox->bottomrighty < {bottomrighty};') + + output = query_parser(query) + + items = [item for item in output] + items = set(items) + + return len(items), items, Image.fromarray(img) + + +def get_visitor_path(person_id, client): + """ + This API gets a visitor's path given the tracker ID. + In order to estimate the visitor's location, we take the midpoint + of the box detected. + """ + query = client.query(f'select \ + object->bbox->topleftx, \ + object->bbox->bottomrightx, \ + object->bbox->toplefty, \ + object->bbox->bottomrighty \ + from {config.ksql_stream_name} \ + where object->id=\'{person_id}\';') + + output = query_parser(query) + + path = [] + + frames = [] + + # Iterate through all the rows and store the coordinates of the bbox + for item in output: + img = cv2.imread("background.png") + item = ast.literal_eval(item) + topleftx = item["row"]["columns"][0] + bottomrightx = item["row"]["columns"][1] + toplefty = item["row"]["columns"][2] + bottomrighty = item["row"]["columns"][3] + center_x, center_y = (bottomrightx + topleftx)//2, bottomrighty + img[center_y-10:center_y+10, center_x-10:center_x+10, :] = 255 + img = cv2.resize(img, dsize=(1280, 720)) + frames.append(img) + path.append([center_x, center_y]) + + # with imageio.get_writer("test.gif", mode="I") as writer: + # for frame in tqdm(frames): + # writer.append_data(frame) + + img = cv2.imread("background.png") + path = np.array(path) + cv2.drawContours(img, [path], 0, (0,0,255), 5) + img = cv2.resize(img, (1280, 720)) + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + cv2.imwrite("testImg.png", img) + + return path, Image.fromarray(img) + + +def get_multiple_visitor_path(person_ids, client): + """ + API to fetch the paths of multiple visitors. + Example use case: + To track a parent and child in the store and identify where the parent and child split up + """ + img = cv2.imread("background.png") + colors = [(0, 0, 255), (0, 255, 0), (255, 0, 0)] + for id in person_ids: + path, _ = get_visitor_path(id, client) + cv2.drawContours(img, [path], 0, (0,0,255), 5) + img = cv2.resize(img, (1280, 720)) + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + cv2.imwrite("multiple.png", img) + return + + +def get_store_heatmap(client, start_time=None, end_time=None): + """ + Function to get a heatmap of the store. + """ + img = plt.imread("background.png") + fig, ax = plt.subplots(2, 1) + ax[0].imshow(img, extent=(0, 2560, 0, 1440)) + query = client.query(f'select \ + object->bbox->topleftx, \ + object->bbox->bottomrightx, \ + object->bbox->toplefty, \ + object->bbox->bottomrighty \ + from {config.ksql_stream_name}') + + output = query_parser(query) + + coordinates = [] + + for item in output: + item = ast.literal_eval(item) + topleftx = item["row"]["columns"][0] + bottomrightx = item["row"]["columns"][1] + toplefty = item["row"]["columns"][2] + bottomrighty = item["row"]["columns"][3] + coordinates.append([(bottomrightx + topleftx)//2, (bottomrighty + toplefty)//2]) + + coordinates = np.array(coordinates) + ax[1].hist2d(coordinates[:, 0], coordinates[:, 1], range=[[0, 2560], [0, 1440]]) + # plt.colorbar() + fig.savefig("heatmap.png", dpi=100) + return + + +def get_basket_pie(client): + """ + Function to get a pie chart of the number of people holding baskets vs no basket + """ + query = client.query(f'select object->obj_prop->hasBasket, \ + object->id \ + from {config.ksql_stream_name};') + output = query_parser(query) + + counts = {} + + for item in output: + item = ast.literal_eval(item) + id = item["row"]["columns"][1] + basket_class = item["row"]["columns"][0] + counts.__setitem__(id, basket_class) + + results = {'hasBasket':0, 'noBasket':0} + + for value in counts.values(): + results[value] += 1 + + return results + + +def get_time_plot(client): + timeframe = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%d") + + query = client.query(f'select timestamp, object->id from {config.ksql_stream_name} \ + where rowtime > \'{timeframe}\'') + results = [] + + output = query_parser(query) + + for item in output: + item = ast.literal_eval(item) + timestamp = pd.to_datetime(item["row"]["columns"][0]).round('1h').strftime('%Y-%m-%d %H:%M:%S') + id = item["row"]["columns"][1] + results.append({"TIMESTAMP":timestamp, "ID":id}) + + df = pd.DataFrame(results, columns=["TIMESTAMP", "ID"]) + + target_df = ( + df.groupby('TIMESTAMP') + .agg(COUNT_PERSONID=('ID', 'nunique')) + .reset_index() + ) + return target_df.to_dict('records') + + +def get_aisle_counts(client): + cfg = ConfigParser() + cfg.read("store_config.ini") + + # Dictionary to store the number of visitors in each aisle + results = [] + + # Read all the sections from the config file + for section in cfg.sections(): + topleftx = cfg.getint(section, "topleftx") + bottomrightx = cfg.getint(section, "bottomrightx") + toplefty = cfg.getint(section, "toplefty") + bottomrighty = cfg.getint(section, "bottomrighty") + num_people, _, _ = get_num_visitors_in_region(topleftx, bottomrightx, + toplefty, bottomrighty, client) + results.append({"aisle": section, "count": num_people}) + + return results + + + +if __name__ == "__main__": + client = KSQLAPI(config.ksql_server) + # print(client.ksql("list streams;")) + # print(get_num_visitors_in_region(50, 1250, 0, 1440, client)) + # get_visitor_path(7, client) + # print(get_num_visitors_time_window(client, "2022-07-13", "2022-07-16")) + # print(get_num_visitors_time_window(client, start_time="2022-07-13")) + # print(get_num_visitors_time_window(client, end_time="2022-07-20")) + # get_store_heatmap(client) + # get_multiple_visitor_path([3, 17], client) + # print(get_basket_pie(client)) + # print(get_time_plot(client)) + # print(get_aisle_counts(client)) \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/migrations/__init__.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/migrations/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/models.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/models.py new file mode 100755 index 0000000..71a8362 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/dashboard.html b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/dashboard.html new file mode 100755 index 0000000..92b6955 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/dashboard.html @@ -0,0 +1,269 @@ + + + + + + + + + + + + + + + + + + + + + +
Number of Visitors Today
+ +
+ + + + + + + + + + + {%block cust_count%} + + {%endblock cust_count%} + +
People w & w/o basketsAisle CountsNumber of Customers in the last hour

{{customer_count}}

+ + + + + + + + \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/path_view.html b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/path_view.html new file mode 100755 index 0000000..6b853e9 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/path_view.html @@ -0,0 +1,9 @@ + + + {%block content %} +
+ +
+ {%endblock content%} + + \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/region_view.html b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/region_view.html new file mode 100755 index 0000000..dc696bf --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/region_view.html @@ -0,0 +1,18 @@ + + + {%block content %} +
+

Number of visitors: {{num_visitors}}

+
+ +
+

Visitor IDs: {{visitors}}

+
+ +
+

Region of Interest

+ +
+ {%endblock content%} + + \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/time_view.html b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/time_view.html new file mode 100755 index 0000000..1d97064 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/time_view.html @@ -0,0 +1,13 @@ + + + {%block content %} +
+

{{num_visitors}}

+
+ +
+

Visitor IDs: {{visitors}}

+
+ {%endblock content%} + + \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/tests.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/tests.py new file mode 100755 index 0000000..7ce503c --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/views.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/views.py new file mode 100755 index 0000000..47bca3f --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/views.py @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json +import datetime +from io import BytesIO +from django.http import HttpResponse +from django.shortcuts import render +from .ksqldb_connecter import get_num_visitors_in_region, get_visitor_path +from .ksqldb_connecter import get_num_visitors_time_window +from .ksqldb_connecter import get_basket_pie +from .ksqldb_connecter import get_time_plot, get_aisle_counts +from ksql import KSQLAPI + +import config +client = KSQLAPI(config.ksql_server) + + +def img_to_base64_str(img): + """ + Helper function to convert PIL image to base64 for rendering + on the webpage + """ + buffered = BytesIO() + img.save(buffered, format="PNG") + buffered.seek(0) + img_byte = buffered.getvalue() + img_str = "data:image/png;base64," + base64.b64encode(img_byte).decode() + return img_str + + +def home(request): + """ + Function to return the dashboard view. + This function calls multiple APIs that return data + * to generate a pie showing people w & w/o baskets + * to generate a plot showing how many people were in the store vs time of the day + * to generate a bar showing how many people were in each aisle + """ + now = datetime.datetime.now() + delta = (now - datetime.timedelta(hours=10)).strftime("%Y-%m-%dT%H:%M:%S") + now = now.strftime("%Y-%m-%dT%H:%M:%S") + basket_counts = get_basket_pie(client) + time_bar = get_time_plot(client) + aisle_counts = get_aisle_counts(client) + customer_count, _ = get_num_visitors_time_window(client, + start_time=delta, + end_time=now) + return render(request, "dashboard.html", context={"basket_counts":json.dumps(basket_counts), + "time_bar": json.dumps(time_bar), + "aisle_counts": aisle_counts, + "customer_count": customer_count}) + + +def num_visitors_in_region_view(request): + if request.method == "GET": + topleftx = request.GET.get("topleftx") + toplefty = request.GET.get("toplefty") + bottomrightx = request.GET.get("bottomrightx") + bottomrighty = request.GET.get("bottomrighty") + coordinates = [topleftx, toplefty, bottomrightx, bottomrighty] + if None in coordinates: + return HttpResponse("Unsupported request format. Use GET request with \ + topleftx, toplefty, bottomrightx, bottomrighty params") + + num_visitors, visitor_ids, roi = get_num_visitors_in_region(topleftx, + bottomrightx, + toplefty, + bottomrighty, client) + + print(visitor_ids) + + roi = img_to_base64_str(roi) + + # return HttpResponse(num_visitors) + return render(request, "region_view.html", context={"num_visitors":num_visitors, + "visitors":visitor_ids, + "img":roi}) + + return HttpResponse("Unsupported request format. Use GET request with \ + topleftx, toplefty, bottomrightx, bottomrighty params") + + +def visitor_path_view(request): + """ + Function to generate a visitor's path given the tracker ID + """ + if request.method == "GET": + person_id = request.GET.get("person_id") + path, img = get_visitor_path(person_id, client) + img = img_to_base64_str(img) + return render(request, "path_view.html", context={"img":img}) + + return HttpResponse("Invalid request format") + + +def num_visitors_in_time_window_view(request): + if request.method == "GET": + start_time = request.GET.get("start_time") + end_time = request.GET.get("end_time") + num_visitors, visitor_ids = get_num_visitors_time_window(client, start_time, end_time) + # return HttpResponse(num_visitors) + return render(request, "time_view.html", context={"num_visitors":num_visitors, + "visitors":visitor_ids}) + + return HttpResponse("Unsupported request format. Use GET request with at least \ + start_time and end_time") diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/background.png b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/background.png new file mode 100755 index 0000000..02f83e1 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/background.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:008c19ae97a60f74d5cc54c1dee089d9fe8f928aa12d3ebb671b6b272038d799 +size 6841022 diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/config.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/config.py new file mode 100755 index 0000000..40b9610 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/config.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set TEST_MODE to false to show dashboard metrics from actual data +TEST_MODE = True + +ksql_server = "http://localhost:8088" +kafka_server = "localhost:9092" + +# test_stream is a dummy stream created to read/write generated data +# test_topic is a dummy kafka topic created to read/write generated data +# This is done to separate actual data from generated data + +if TEST_MODE: + ksql_stream_name = "test_stream" + kafka_topic = "test_topic" +else: + ksql_stream_name = "detections_stream" + kafka_topic = "detections" diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/environment.yml b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/environment.yml new file mode 100755 index 0000000..7120549 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/environment.yml @@ -0,0 +1,59 @@ +name: web-dev +channels: + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - ca-certificates=2022.07.19=h06a4308_0 + - certifi=2022.6.15=py38h06a4308_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.3=he6710b0_2 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - ncurses=6.3=h5eee18b_3 + - openssl=1.1.1q=h7f8727e_0 + - pip=22.1.2=py38h06a4308_0 + - python=3.8.13=h12debd9_0 + - readline=8.1.2=h7f8727e_1 + - setuptools=61.2.0=py38h06a4308_0 + - sqlite=3.39.2=h5082296_0 + - tk=8.6.12=h1ccaba5_0 + - wheel=0.37.1=pyhd3eb1b0_0 + - xz=5.2.5=h7f8727e_1 + - zlib=1.2.12=h7f8727e_2 + - pip: + - anyio==3.6.1 + - asgiref==3.5.2 + - backports-zoneinfo==0.2.1 + - charset-normalizer==2.1.0 + - confluent-kafka==1.9.2 + - cycler==0.11.0 + - django==4.1 + - fonttools==4.34.4 + - h11==0.12.0 + - h2==4.1.0 + - hpack==4.0.0 + - httpcore==0.15.0 + - httpx==0.23.0 + - hyperframe==6.0.1 + - idna==3.3 + - kiwisolver==1.4.4 + - matplotlib==3.5.2 + - numpy==1.23.1 + - opencv-python==4.6.0.66 + - packaging==21.3 + - pandas==1.4.3 + - pillow==9.2.0 + - pyksql==0.11.0 + - pyparsing==3.0.9 + - python-dateutil==2.8.2 + - pytz==2022.1 + - requests==2.28.1 + - rfc3986==1.5.0 + - six==1.16.0 + - sniffio==1.2.0 + - sqlparse==0.4.2 + - tqdm==4.64.0 + - urllib3==1.26.11 +prefix: /work/anaconda3/envs/web-dev diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/manage.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/manage.py new file mode 100755 index 0000000..cf86fe2 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'retail_iva.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/random_message_generator.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/random_message_generator.py new file mode 100755 index 0000000..e2a89c4 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/random_message_generator.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from random import randrange, choice +import datetime +import json +from uuid import uuid4 +from confluent_kafka import Producer +from config import kafka_topic, kafka_server +from tqdm import tqdm + +def generate_random_timestamp(start): + ts = (start - datetime.timedelta(hours=randrange(12), + minutes=randrange(60), + seconds=randrange(60))).strftime("%Y-%m-%dT%H:%M:%S.%f") + return ts[:-3] + 'Z' + + +def generate_random_bbox(max_width=2560, max_height=1440): + bottomrightx = randrange(max_width) + topleftx = randrange(bottomrightx) + bottomrighty = randrange(max_height) + toplefty = randrange(bottomrighty) + return {"topleftx": topleftx, + "toplefty": toplefty, + "bottomrightx": bottomrightx, + "bottomrighty": bottomrighty} + + +def generate_random_obj_id(max_id=10): + return str(randrange(max_id)) + + +def generate_random_basket_des(classes=["hasBasket", "noBasket"]): + return choice(classes) + + +def generate_kafka_message(): + message = {} + # Set messageid and mdsversion + message.__setitem__("messageid", str(uuid4())) + message.__setitem__("mdsversion", "1.0") + + # Set timestamp + message.__setitem__("timestamp", generate_random_timestamp(datetime.datetime.now())) + + # Create and set object + object = {"id":generate_random_obj_id(), + "speed": 0, + "direction": 0, + "orientation": 0, + "detection": "person", + "obj_prop": {"hasBasket": generate_random_basket_des(), + "confidence": 0.99}, + "bbox": generate_random_bbox()} + message.__setitem__("object", object) + + # Set event_des and videopath + message.__setitem__("event_des", {"id": str(uuid4()), "type": "entry"}) + message.__setitem__("videopath", "") + return message + + +if __name__ == "__main__": + if kafka_topic == "detections": + print("WARNING: Writing messages to main topic. Set TEST_MODE to True in config.py to write messages to dummy topic") + + # Define a producer for the topic + prd = Producer({"bootstrap.servers":kafka_server}) + + # Write messages to the topic + for _ in tqdm(range(1)): + msg = generate_kafka_message() + print(msg) + prd.produce(topic=kafka_topic, value=json.dumps(msg, indent=4)) + + # Block until messages are sent + prd.poll(10000) + prd.flush() diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/requirements.txt b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/requirements.txt new file mode 100755 index 0000000..c4bceba --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/requirements.txt @@ -0,0 +1,35 @@ +anyio==3.6.1 +asgiref==3.5.2 +backports.zoneinfo==0.2.1 +charset-normalizer==2.1.0 +cycler==0.11.0 +distlib==0.3.4 +Django==4.1 +filelock==3.7.0 +fonttools==4.34.4 +h11==0.12.0 +h2==4.1.0 +hpack==4.0.0 +httpcore==0.15.0 +httpx==0.23.0 +hyperframe==6.0.1 +idna==3.3 +kiwisolver==1.4.4 +matplotlib==3.5.2 +numpy==1.23.1 +opencv-python==4.6.0.66 +packaging==21.3 +pandas==1.4.3 +Pillow==9.2.0 +platformdirs==2.5.2 +pykSQL==0.11.0 +pyparsing==3.0.9 +python-dateutil==2.8.2 +pytz==2022.1 +requests==2.28.1 +rfc3986==1.5.0 +six==1.16.0 +sniffio==1.2.0 +sqlparse==0.4.2 +tqdm==4.64.0 +urllib3==1.26.11 diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/asgi.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/asgi.py new file mode 100755 index 0000000..90d6016 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for retail_iva project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'retail_iva.settings') + +application = get_asgi_application() diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/settings.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/settings.py new file mode 100755 index 0000000..29e9279 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/settings.py @@ -0,0 +1,124 @@ +""" +Django settings for retail_iva project. + +Generated by 'django-admin startproject' using Django 4.0.5. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.0/ref/settings/ +""" + +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'django-insecure-y%*c((yh(%l_$&7z+$ei^40l4l&nkuhdxfdb^=mfabl53bc&*=' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = ["*"] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'analytics' +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'retail_iva.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'retail_iva.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/4.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db.sqlite3', + } +} + + +# Password validation +# https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.0/howto/static-files/ + +STATIC_URL = 'static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/urls.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/urls.py new file mode 100755 index 0000000..3829e49 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/urls.py @@ -0,0 +1,26 @@ +"""retail_iva URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path +from analytics import views + +urlpatterns = [ + path('admin/', admin.site.urls), + path('', views.home), + path('num-visitors-region/', views.num_visitors_in_region_view), + path('num-visitors-time/', views.num_visitors_in_time_window_view), + path('visitor-path/', views.visitor_path_view), +] diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/wsgi.py b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/wsgi.py new file mode 100755 index 0000000..1fa1c8c --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for retail_iva project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'retail_iva.settings') + +application = get_wsgi_application() diff --git a/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/store_config.ini b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/store_config.ini new file mode 100755 index 0000000..2e00b7e --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/store_config.ini @@ -0,0 +1,17 @@ +[DEFAULT] +topleftx = 0 +bottomrightx = 0 +toplefty = 0 +bottomrighty = 0 + +[Aisle1] +topleftx = 50 +bottomrightx = 1400 +toplefty = 0 +bottomrighty = 1440 + +[Aisle2] +topleftx = 1401 +bottomrightx = 2560 +toplefty = 0 +bottomrighty = 1440 \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/files/README.md b/legacy_apps/deepstream-retail-analytics/files/README.md new file mode 100755 index 0000000..10d99a1 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/files/README.md @@ -0,0 +1,21 @@ +# Description of files + +* [0001_compressed.h264](./0001_compressed.h264) - Input file for running the pipeline +* [basketClassifier.etlt](./basketClassifier.etlt) - Model file for classifying people with and without baskets +* [basket_classifier_labels.txt](./basket_classifier_labels.txt) - Label file for the above model + +# RECOMMENDED: Verify checksum after cloning files using git LFS + +LFS files are not cloned by default when you clone the repository + +Clone model and input files by running + +```bash +git lfs pull +``` + +Verify the checksum of files after cloning + +```bash +sha512sum -c checksum.txt +``` \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/files/basketClassifier.etlt b/legacy_apps/deepstream-retail-analytics/files/basketClassifier.etlt new file mode 100755 index 0000000..919a477 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/files/basketClassifier.etlt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:737ccfda9fbe255a4e5127c80d027f0f2798182133972287ba144535cb1a6a70 +size 85344951 diff --git a/legacy_apps/deepstream-retail-analytics/files/basket_classifier_labels.txt b/legacy_apps/deepstream-retail-analytics/files/basket_classifier_labels.txt new file mode 100755 index 0000000..897557e --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/files/basket_classifier_labels.txt @@ -0,0 +1 @@ +hasBasket;noBasket \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/files/checksum.txt b/legacy_apps/deepstream-retail-analytics/files/checksum.txt new file mode 100755 index 0000000..d7f8d76 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/files/checksum.txt @@ -0,0 +1,2 @@ +73563263b7b11b3c3355ea4af76733f783f75abcd2936d463829fee8ece33dece1c477ddcc437e0eb499f35ecd7bfbd87f8c96a3e41d0b6940f96eabe1b30984 0001_compressed.h264 +1dc119bea04488b095eae0e2c7c963daed2504276ff489c215bb2cb60bb4e9e82a335e2f3b29c17ee3dd5a15403733fd5d5e67fd67f4e9025cc4c86c700fadea basketClassifier.etlt diff --git a/legacy_apps/deepstream-retail-analytics/media/arch.png b/legacy_apps/deepstream-retail-analytics/media/arch.png new file mode 100755 index 0000000..cf71696 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/media/arch.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e428455f05c1281a5f590bffac7ffd1571e17d46a56b929ff72dbdca841074d8 +size 148633 diff --git a/legacy_apps/deepstream-retail-analytics/media/dashboard.png b/legacy_apps/deepstream-retail-analytics/media/dashboard.png new file mode 100755 index 0000000..cfdf214 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/media/dashboard.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dba2950e872e53429c2d5edca0db634d7e31ddb43ef39ac1100e14158129433b +size 83422 diff --git a/legacy_apps/deepstream-retail-analytics/media/deepstream_container.png b/legacy_apps/deepstream-retail-analytics/media/deepstream_container.png new file mode 100755 index 0000000..06f9aaa --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/media/deepstream_container.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90fe03d2e1d427fc48b1ca080f91b8e5b4991fe614f71b7e9c816f6ac89aa6fc +size 13580 diff --git a/legacy_apps/deepstream-retail-analytics/media/docker_container_ps.png b/legacy_apps/deepstream-retail-analytics/media/docker_container_ps.png new file mode 100755 index 0000000..1bfc140 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/media/docker_container_ps.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5be7d9d72ad127003155a2ae7a0d5c90b44c1a6aa7aaba6414b32a1ba1e102f +size 54740 diff --git a/legacy_apps/deepstream-retail-analytics/media/output.gif b/legacy_apps/deepstream-retail-analytics/media/output.gif new file mode 100755 index 0000000..b0e9f80 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/media/output.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f855cc35e5f60ea2b18356e75dcb464467b12dff67120b333ab255edd54f7a0 +size 10103813 diff --git a/legacy_apps/deepstream-retail-analytics/media/retail-iva-arch.png b/legacy_apps/deepstream-retail-analytics/media/retail-iva-arch.png new file mode 100755 index 0000000..37b213a --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/media/retail-iva-arch.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49d6446d06f0b04059d7808bb048a454b6e9df8ca4fc0b57d57422a0db1988c1 +size 149982 diff --git a/legacy_apps/deepstream-retail-analytics/nvdsmeta_schema.h b/legacy_apps/deepstream-retail-analytics/nvdsmeta_schema.h new file mode 100755 index 0000000..14f99f3 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/nvdsmeta_schema.h @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved. + * + * NVIDIA Corporation and its licensors retain all intellectual property + * and proprietary rights in and to this software, related documentation + * and any modifications thereto. Any use, reproduction, disclosure or + * distribution of this software and related documentation without an express + * license agreement from NVIDIA Corporation is strictly prohibited. + * + */ + +/** + * @file + * NVIDIA DeepStream: Metadata Extension Structures + * + * @b Description: This file defines the NVIDIA DeepStream metadata structures + * used to describe metadata objects. + */ + +/** + * @defgroup metadata_extensions Metadata Extension Structures + * + * Defines metadata structures used to describe metadata objects. + * + * @ingroup NvDsMetaApi + * @{ + */ + +#ifndef NVDSMETA_H_ +#define NVDSMETA_H_ + +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * Defines event type flags. + */ +typedef enum NvDsEventType { + NVDS_EVENT_ENTRY, + NVDS_EVENT_EXIT, + NVDS_EVENT_MOVING, + NVDS_EVENT_STOPPED, + NVDS_EVENT_EMPTY, + NVDS_EVENT_PARKED, + NVDS_EVENT_RESET, + + /** Reserved for future use. Custom events must be assigned values + greater than this. */ + NVDS_EVENT_RESERVED = 0x100, + /** Specifies a custom event. */ + NVDS_EVENT_CUSTOM = 0x101, + NVDS_EVENT_FORCE32 = 0x7FFFFFFF +} NvDsEventType; + +/** + * Defines object type flags. + */ +typedef enum NvDsObjectType { + NVDS_OBJECT_TYPE_VEHICLE, + NVDS_OBJECT_TYPE_PERSON, + NVDS_OBJECT_TYPE_FACE, + NVDS_OBJECT_TYPE_BAG, + NVDS_OBJECT_TYPE_BICYCLE, + NVDS_OBJECT_TYPE_ROADSIGN, + NVDS_OBJECT_TYPE_VEHICLE_EXT, + NVDS_OBJECT_TYPE_PERSON_EXT, + NVDS_OBJECT_TYPE_FACE_EXT, + /** Reserved for future use. Custom objects must be assigned values + greater than this. */ + NVDS_OBJECT_TYPE_RESERVED = 0x100, + /** Specifies a custom object. */ + NVDS_OBJECT_TYPE_CUSTOM = 0x101, + /** "object" key will be missing in the schema */ + NVDS_OBJECT_TYPE_UNKNOWN = 0x102, + NVDS_OBEJCT_TYPE_FORCE32 = 0x7FFFFFFF +} NvDsObjectType; + +/** + * Defines payload type flags. + */ +typedef enum NvDsPayloadType { + NVDS_PAYLOAD_DEEPSTREAM, + NVDS_PAYLOAD_DEEPSTREAM_MINIMAL, + /** Reserved for future use. Custom payloads must be assigned values + greater than this. */ + NVDS_PAYLOAD_RESERVED = 0x100, + /** Specifies a custom payload. You must implement the nvds_msg2p_* + interface. */ + NVDS_PAYLOAD_CUSTOM = 0x101, + NVDS_PAYLOAD_FORCE32 = 0x7FFFFFFF +} NvDsPayloadType; + +/** + * Holds a rectangle's position and size. + */ +typedef struct NvDsRect { + float top; /**< Holds the position of rectangle's top in pixels. */ + float left; /**< Holds the position of rectangle's left side in pixels. */ + float width; /**< Holds the rectangle's width in pixels. */ + float height; /**< Holds the rectangle's height in pixels. */ +} NvDsRect; + +/** + * Holds geolocation parameters. + */ +typedef struct NvDsGeoLocation { + gdouble lat; /**< Holds the location's latitude. */ + gdouble lon; /**< Holds the location's longitude. */ + gdouble alt; /**< Holds the location's altitude. */ +} NvDsGeoLocation; + +/** + * Hold a coordinate's position. + */ +typedef struct NvDsCoordinate { + gdouble x; /**< Holds the coordinate's X position. */ + gdouble y; /**< Holds the coordinate's Y position. */ + gdouble z; /**< Holds the coordinate's Z position. */ +} NvDsCoordinate; + +/** + * Holds an object's signature. + */ +typedef struct NvDsObjectSignature { + /** Holds a pointer to an array of signature values. */ + gdouble *signature; + /** Holds the number of signature values in @a signature. */ + guint size; +} NvDsObjectSignature; + +/** + * Holds a vehicle object's parameters. + */ +typedef struct NvDsVehicleObject { + gchar *type; /**< Holds a pointer to the type of the vehicle. */ + gchar *make; /**< Holds a pointer to the make of the vehicle. */ + gchar *model; /**< Holds a pointer to the model of the vehicle. */ + gchar *color; /**< Holds a pointer to the color of the vehicle. */ + gchar *region; /**< Holds a pointer to the region of the vehicle. */ + gchar *license; /**< Holds a pointer to the license number of the vehicle.*/ +} NvDsVehicleObject; + +/** + * Holds a person object's parameters. + */ +typedef struct NvDsPersonObject { + gchar *gender; /**< Holds a pointer to the person's gender. */ + gchar *hair; /**< Holds a pointer to the person's hair color. */ + gchar *cap; /**< Holds a pointer to the type of cap the person is + wearing, if any. */ + gchar *apparel; /**< Holds a pointer to a description of the person's + apparel. */ + guint age; /**< Holds the person's age. */ + // Modified the stock NvDsPersonObject that comes with DeepStream 6.1 + gchar *hasBasket; +} NvDsPersonObject; + +/** + * Holds a face object's parameters. + */ +typedef struct NvDsFaceObject { + gchar *gender; /**< Holds a pointer to the person's gender. */ + gchar *hair; /**< Holds a pointer to the person's hair color. */ + gchar *cap; /**< Holds a pointer to the type of cap the person + is wearing, if any. */ + gchar *glasses; /**< Holds a pointer to the type of glasses the person + is wearing, if any. */ + gchar *facialhair;/**< Holds a pointer to the person's facial hair color. */ + gchar *name; /**< Holds a pointer to the person's name. */ + gchar *eyecolor; /**< Holds a pointer to the person's eye color. */ + guint age; /**< Holds the person's age. */ +} NvDsFaceObject; + +/** + * Holds a vehicle object's parameters. + */ +typedef struct NvDsVehicleObjectExt { + gchar *type; /**< Holds a pointer to the type of the vehicle. */ + gchar *make; /**< Holds a pointer to the make of the vehicle. */ + gchar *model; /**< Holds a pointer to the model of the vehicle. */ + gchar *color; /**< Holds a pointer to the color of the vehicle. */ + gchar *region; /**< Holds a pointer to the region of the vehicle. */ + gchar *license; /**< Holds a pointer to the license number of the vehicle.*/ + + GList *mask; /**< Holds a list of polygons for vehicle mask. */ +} NvDsVehicleObjectExt; + +/** + * Holds a person object's parameters. + */ +typedef struct NvDsPersonObjectExt { + gchar *gender; /**< Holds a pointer to the person's gender. */ + gchar *hair; /**< Holds a pointer to the person's hair color. */ + gchar *cap; /**< Holds a pointer to the type of cap the person is + wearing, if any. */ + gchar *apparel; /**< Holds a pointer to a description of the person's + apparel. */ + guint age; /**< Holds the person's age. */ + + GList *mask; /**< Holds a list of polygons for person mask. */ +} NvDsPersonObjectExt; + +/** + * Holds a face object's parameters. + */ +typedef struct NvDsFaceObjectWithExt { + gchar *gender; /**< Holds a pointer to the person's gender. */ + gchar *hair; /**< Holds a pointer to the person's hair color. */ + gchar *cap; /**< Holds a pointer to the type of cap the person + is wearing, if any. */ + gchar *glasses; /**< Holds a pointer to the type of glasses the person + is wearing, if any. */ + gchar *facialhair;/**< Holds a pointer to the person's facial hair color. */ + gchar *name; /**< Holds a pointer to the person's name. */ + gchar *eyecolor; /**< Holds a pointer to the person's eye color. */ + guint age; /**< Holds the person's age. */ + + GList *mask; /**< Holds a list of polygons for face mask. */ +} NvDsFaceObjectExt; + +/** + * Holds event message meta data. + * + * You can attach various types of objects (vehicle, person, face, etc.) + * to an event by setting a pointer to the object in @a extMsg. + * + * Similarly, you can attach a custom object to an event by setting a pointer to the object in @a extMsg. + * A custom object must be handled by the metadata parsing module accordingly. + */ +typedef struct NvDsEventMsgMeta { + /** Holds the event's type. */ + NvDsEventType type; + /** Holds the object's type. */ + NvDsObjectType objType; + /** Holds the object's bounding box. */ + NvDsRect bbox; + /** Holds the object's geolocation. */ + NvDsGeoLocation location; + /** Holds the object's coordinates. */ + NvDsCoordinate coordinate; + /** Holds the object's signature. */ + NvDsObjectSignature objSignature; + /** Holds the object's class ID. */ + gint objClassId; + /** Holds the ID of the sensor that generated the event. */ + gint sensorId; + /** Holds the ID of the analytics module that generated the event. */ + gint moduleId; + /** Holds the ID of the place related to the object. */ + gint placeId; + /** Holds the ID of the component (plugin) that generated this event. */ + gint componentId; + /** Holds the video frame ID of this event. */ + gint frameId; + /** Holds the confidence level of the inference. */ + gdouble confidence; + /** Holds the object's tracking ID. */ + guint64 trackingId; + /** Holds a pointer to the generated event's timestamp. */ + gchar *ts; + /** Holds a pointer to the detected or inferred object's ID. */ + gchar *objectId; + + /** Holds a pointer to a string containing the sensor's identity. */ + gchar *sensorStr; + /** Holds a pointer to a string containing other attributes associated with + the object. */ + gchar *otherAttrs; + /** Holds a pointer to the name of the video file. */ + gchar *videoPath; + /** Holds a pointer to event message meta data. This can be used to hold + data that can't be accommodated in the existing fields, or an associated + object (representing a vehicle, person, face, etc.). */ + gpointer extMsg; + /** Holds the size of the custom object at @a extMsg. */ + guint extMsgSize; +} NvDsEventMsgMeta; + +/** + * Holds event information. + */ +typedef struct _NvDsEvent { + /** Holds the type of event. */ + NvDsEventType eventType; + /** Holds a pointer to event metadata. */ + NvDsEventMsgMeta *metadata; +} NvDsEvent; + +/** + * Holds data for any user defined custom message to be attached to the payload + * message : custom message to be attached + * size : size of the custom message + */ +typedef struct _NvDsCustomMsgInfo { + void *message; + guint size; +}NvDsCustomMsgInfo; + +/** + * Holds payload metadata. + */ +typedef struct NvDsPayload { + /** Holds a pointer to the payload. */ + gpointer payload; + /** Holds the size of the payload. */ + guint payloadSize; + /** Holds the ID of the component (plugin) which attached the payload + (optional). */ + guint componentId; +} NvDsPayload; + +#ifdef __cplusplus +} +#endif +#endif /* NVDSMETA_H_ */ + +/** @} */ diff --git a/legacy_apps/deepstream-retail-analytics/nvmsgconv/Makefile b/legacy_apps/deepstream-retail-analytics/nvmsgconv/Makefile new file mode 100755 index 0000000..9d47b47 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/nvmsgconv/Makefile @@ -0,0 +1,48 @@ +############################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +CC:= g++ + +PKGS:= glib-2.0 gobject-2.0 json-glib-1.0 uuid + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + +CFLAGS:= -Wall -std=c++11 -shared -fPIC + +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes -I./deepstream_schema + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) +LIBS:= $(shell pkg-config --libs $(PKGS)) + +LIBS+= -lyaml-cpp + +SRCFILES:= nvmsgconv.cpp \ + deepstream_schema/eventmsg_payload_peoplenet.cpp \ + deepstream_schema/dsmeta_payload.cpp \ + deepstream_schema/deepstream_schema.cpp +TARGET_LIB:= libnvds_msgconv.so + +all: $(TARGET_LIB) + +$(TARGET_LIB) : $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS) $(LIBS) + +install: $(TARGET_LIB) + cp -rv $(TARGET_LIB) $(LIB_INSTALL_DIR) + +clean: + rm -rf $(TARGET_LIB) diff --git a/legacy_apps/deepstream-retail-analytics/nvmsgconv/README.md b/legacy_apps/deepstream-retail-analytics/nvmsgconv/README.md new file mode 100755 index 0000000..5004b78 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/nvmsgconv/README.md @@ -0,0 +1,124 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +Refer to the DeepStream SDK documentation for a description of the plugin. + +-------------------------------------------------------------------------------- +Pre-requisites: +- glib-2.0 +- json-glib-1.0 +- uuid +- yaml-cpp + +Install using: + sudo apt-get install libglib2.0-dev libjson-glib-dev uuid-dev libyaml-cpp-dev + +-------------------------------------------------------------------------------- +Compiling and installing the plugin: +Run make and sudo make install + +NOTE: To compile the sources, run make with "sudo" or root permission. + +# How to Modify NvMsgConv Plugin? + +The schema for the kafka messages sent by the DS app is defined in the [eventmsg_payload.cpp](./deepstream_schema/eventmsg_payload.cpp) file. + +In this example, we will demonstrate how to build a custom message converter library to add custom message information to the message payload. + +**Why should we modify the default library file?** + +* Notice the [`NvDsPersonObject`](https://docs.nvidia.com/metropolis/deepstream/sdk-api/structNvDsPersonObject.html) struct provided in the `nvdsmeta_schema.h` file. The struct by default has the ability to carry only information about gender, age etc. + +* The message schema for this project is different from the default schema that comes with DeepStream SDK. In order to generate this new schema, we need to modify the + +## Step 1 - Modify NvDsPersonObject + +For this use case, we would like to modify the object so that it has an additional attribute [`hasBasket`](../nvdsmeta_schema.h#L159). To achieve this we modify the existing struct in `/opt/nvidia/deepstream/deepstream/sources/includes/nvdsmeta_schema.h` as below. + +```c +typedef struct NvDsPersonObject { + gchar *gender; /**< Holds a pointer to the person's gender. */ + gchar *hair; /**< Holds a pointer to the person's hair color. */ + gchar *cap; /**< Holds a pointer to the type of cap the person is + wearing, if any. */ + gchar *apparel; /**< Holds a pointer to a description of the person's + apparel. */ + guint age; /**< Holds the person's age. */ + // Modified the stock NvDsPersonObject that comes with DeepStream 6.1 + gchar *hasBasket; +} NvDsPersonObject; +``` + +## Step 2 - Modify library file for NvMsgConv + +Original file that comes with DeepStream SDK - [eventmsg_payload.cpp](./deepstream_schema/eventmsg_payload.cpp) + +Modified file for current use case - [eventmsg_payload_peoplenet.cpp](./deepstream_schema/eventmsg_payload_peoplenet.cpp) + +It is a good exercise to compare the two files to understand how to modify the original library file. + +Sample message generated by the new message library + +```json +{ + "messageid": "50e5a5f5-0568-4d45-9b97-4bcd5ee695be", + "mdsversion": "1.0", + "timestamp": "2022-08-17T04:11:15.074Z", + "object": { + "id": "8", + "speed": 0, + "direction": 0, + "orientation": 0, + "detection": "person", + "obj_prop": { + "hasBasket": "hasBasket", + "confidence": 0.99 + }, + "bbox": { + "topleftx": 1877, + "toplefty": 52, + "bottomrightx": 2524, + "bottomrighty": 101 + } + }, + "event_des": { + "id": "87ad972c-175b-4eb9-9012-11999314a7e7", + "type": "entry" + }, + "videopath": "" +} +``` + +* The default library message converter plugin has fields for sensor, place, analytics data. Since we are not using any of those keys for the current project, we can remove the functions that generate those objects in the payload. + +* We are concerned only with `NvDsPersonObject` for this project. We can safely remove the message creation functions for all other classes. + +* You can set the newly added attribute `hasBasket` by adding the following line to [`generate_object_object`](./deepstream_schema/eventmsg_payload_peoplenet.cpp#L72) function + +```cpp +json_object_set_string_member (jobject, "hasBasket", dsObj->hasBasket); +``` + +## Step 3 - Compiling the C++ file to generate the library file + +* Modify the path in [`Makefile`](./Makefile) to reflect the path of the new C++ file + +* Run `make -B` to compile all the codes and generate the new library + +* You can find the new library file in the same directory - [libnvds_msgconv.so](./libnvds_msgconv.so) + +* Add the path for the newly compiled library in the [`msgconv`](../configs/retail_iva.yml#L49) section of the main config file \ No newline at end of file diff --git a/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/deepstream_schema.cpp b/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/deepstream_schema.cpp new file mode 100755 index 0000000..25077fd --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/deepstream_schema.cpp @@ -0,0 +1,789 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +#include "deepstream_schema.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +static void +get_csv_tokens (const string &text, vector &tokens) +{ + /* This is based on assumption that fields and their locations + * are fixed in CSV file. This should be updated accordingly if + * that is not the case. + */ + gint count = 0; + + gchar **csv_tokens = g_strsplit (text.c_str(), ",", -1); + gchar **temp = csv_tokens; + gchar *token; + + while (*temp && count < DEFAULT_CSV_FIELDS) { + token = *temp++; + tokens.push_back (string(g_strstrip(token))); + count++; + } + g_strfreev (csv_tokens); +} + +static bool +nvds_msg2p_parse_sensor (void *privData, GKeyFile *key_file, gchar *group) +{ + bool ret = false; + bool isEnabled = false; + gchar **keys = NULL; + gchar **key = NULL; + GError *error = NULL; + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject sensorObj; + gint sensorId; + gchar *keyVal; + + + if (sscanf (group, CONFIG_GROUP_SENSOR "%u", &sensorId) < 1) { + cout << "Wrong sensor group name " << group << endl; + return ret; + } + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->sensorObj.find (sensorId); + if (idMap != privObj->sensorObj.end()) { + cout << "Duplicate entries for " << group << endl; + return ret; + } + + isEnabled = g_key_file_get_boolean (key_file, group, CONFIG_KEY_ENABLE, + &error); + if (!isEnabled) { + // Not enabled, skip the parsing of keys. + ret = true; + goto done; + } else { + g_key_file_remove_key (key_file, group, CONFIG_KEY_ENABLE, + &error); + CHECK_ERROR (error); + } + + keys = g_key_file_get_keys (key_file, group, NULL, &error); + CHECK_ERROR (error); + + for (key = keys; *key; key++) { + keyVal = NULL; + if (!g_strcmp0 (*key, CONFIG_KEY_ID)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_ID, &error); + sensorObj.id = keyVal; + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_TYPE)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_TYPE, &error); + sensorObj.type = keyVal; + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_DESCRIPTION)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_DESCRIPTION, &error); + sensorObj.desc = keyVal; + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_LOCATION)) { + gsize length; + gdouble *location = g_key_file_get_double_list (key_file, group, + CONFIG_KEY_LOCATION, + &length, &error); + if (length != 3) { + cout << "Wrong values provided, it should be like lat;lon;alt" << endl; + g_free (location); + goto done; + } + + memcpy (sensorObj.location, location, length * sizeof (gdouble)); + g_free (location); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_COORDINATE)) { + gsize length; + gdouble *coordinate = g_key_file_get_double_list (key_file, group, + CONFIG_KEY_COORDINATE, + &length, &error); + if (length != 3) { + cout << "Wrong values provided, it should be like x;y;z" << endl; + g_free (coordinate); + goto done; + } + + memcpy (sensorObj.coordinate, coordinate, length * sizeof (gdouble)); + g_free (coordinate); + CHECK_ERROR (error); + } else { + cout << "Unknown key " << *key << " for group [" << group <<"]\n"; + } + + if (keyVal) + g_free (keyVal); + } + + privObj->sensorObj.insert (make_pair (sensorId, sensorObj)); + + ret = true; + +done: + if (error) { + g_error_free (error); + } + if (keys) { + g_strfreev (keys); + } + + return ret; +} + +static bool +nvds_msg2p_parse_place (void *privData, GKeyFile *key_file, gchar *group) +{ + bool ret = false; + bool isEnabled = false; + gchar **keys = NULL; + gchar **key = NULL; + GError *error = NULL; + NvDsPayloadPriv *privObj = NULL; + NvDsPlaceObject placeObj; + gint placeId; + gchar *keyVal; + + if (sscanf (group, CONFIG_GROUP_PLACE "%u", &placeId) < 1) { + cout << "Wrong place group name " << group << endl; + return ret; + } + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->placeObj.find (placeId); + if (idMap != privObj->placeObj.end()) { + cout << "Duplicate entries for " << group << endl; + return ret; + } + + isEnabled = g_key_file_get_boolean (key_file, group, CONFIG_KEY_ENABLE, + &error); + if (!isEnabled) { + // Not enabled, skip the parsing of keys. + ret = true; + goto done; + } else { + g_key_file_remove_key (key_file, group, CONFIG_KEY_ENABLE, + &error); + CHECK_ERROR (error); + } + + keys = g_key_file_get_keys (key_file, group, NULL, &error); + CHECK_ERROR (error); + + for (key = keys; *key; key++) { + if (!g_strcmp0 (*key, CONFIG_KEY_ID)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_ID, &error); + placeObj.id = keyVal; + g_free (keyVal); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_TYPE)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_TYPE, &error); + placeObj.type = keyVal; + g_free (keyVal); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_NAME)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_NAME, &error); + placeObj.name = keyVal; + g_free (keyVal); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_LOCATION)) { + gsize length; + gdouble *location = g_key_file_get_double_list (key_file, group, + CONFIG_KEY_LOCATION, + &length, &error); + if (length != 3) { + cout << "Wrong values provided, it should be like lat;lon;alt" << endl; + g_free (location); + goto done; + } + + memcpy (placeObj.location, location, length * sizeof (gdouble)); + g_free (location); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_COORDINATE)) { + gsize length; + gdouble *coordinate = g_key_file_get_double_list (key_file, group, + CONFIG_KEY_COORDINATE, + &length, &error); + if (length != 3) { + cout << "Wrong values provided, it should be like x;y;z" << endl; + g_free (coordinate); + goto done; + } + + memcpy (placeObj.coordinate, coordinate, length * sizeof (gdouble)); + g_free (coordinate); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_PLACE_SUB_FIELD1)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_PLACE_SUB_FIELD1, &error); + placeObj.subObj.field1 = keyVal; + g_free (keyVal); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_PLACE_SUB_FIELD2)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_PLACE_SUB_FIELD2, &error); + placeObj.subObj.field2 = keyVal; + g_free (keyVal); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_PLACE_SUB_FIELD3)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_PLACE_SUB_FIELD3, &error); + placeObj.subObj.field3 = keyVal; + g_free (keyVal); + CHECK_ERROR (error); + } else { + cout << "Unknown key " << *key << " for group [" << group <<"]\n"; + } + } + + privObj->placeObj.insert (pair (placeId, placeObj)); + + ret = true; + +done: + if (error) { + g_error_free (error); + } + if (keys) { + g_strfreev (keys); + } + + return ret; +} + +static bool +nvds_msg2p_parse_analytics (void *privData, GKeyFile *key_file, gchar *group) +{ + bool ret = false; + bool isEnabled = false; + gchar **keys = NULL; + gchar **key = NULL; + GError *error = NULL; + NvDsPayloadPriv *privObj = NULL; + NvDsAnalyticsObject analyticsObj; + gint moduleId; + gchar *keyVal; + + if (sscanf (group, CONFIG_GROUP_ANALYTICS "%u", &moduleId) < 1) { + cout << "Wrong analytics module group name " << group << endl; + return ret; + } + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->analyticsObj.find (moduleId); + if (idMap != privObj->analyticsObj.end()) { + cout << "Duplicate entries for " << group << endl; + return ret; + } + + isEnabled = g_key_file_get_boolean (key_file, group, CONFIG_KEY_ENABLE, + &error); + if (!isEnabled) { + // Not enabled, skip the parsing of keys. + ret = true; + goto done; + } else { + g_key_file_remove_key (key_file, group, CONFIG_KEY_ENABLE, + &error); + CHECK_ERROR (error); + } + + keys = g_key_file_get_keys (key_file, group, NULL, &error); + CHECK_ERROR (error); + + for (key = keys; *key; key++) { + keyVal = NULL; + if (!g_strcmp0 (*key, CONFIG_KEY_ID)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_ID, &error); + analyticsObj.id = keyVal; + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_SOURCE)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_SOURCE, &error); + analyticsObj.source = keyVal; + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_DESCRIPTION)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_DESCRIPTION, &error); + analyticsObj.desc = keyVal; + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_VERSION)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_VERSION, &error); + analyticsObj.version = keyVal; + CHECK_ERROR (error); + } else { + cout << "Unknown key " << *key << " for group [" << group <<"]\n"; + } + + if (keyVal) + g_free (keyVal); + } + + privObj->analyticsObj.insert (make_pair (moduleId, analyticsObj)); + + ret = true; + +done: + if (error) { + g_error_free (error); + } + if (keys) { + g_strfreev (keys); + } + + return ret; +} + +/* Separate a config file entry with delimiters + * into strings. */ +static std::vector +split_string (std::string input) { + std::vector positions; + for (unsigned int i = 0; i < input.size(); i++) { + if (input[i] == ';') + positions.push_back(i); + } + std::vector ret; + int prev = 0; + for (auto &j: positions) { + std::string temp = input.substr(prev, j - prev); + ret.push_back(temp); + prev = j + 1; + } + ret.push_back(input.substr(prev, input.size() - prev)); + return ret; +} + +static bool +nvds_msg2p_parse_sensor_yaml (void *privData, gchar *cfg_file_path, std::string group_str) +{ + bool ret = false; + bool isEnabled = false; + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject sensorObj; + gint sensorId; + + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + if (sscanf (group_str.c_str(), CONFIG_GROUP_SENSOR "%u", &sensorId) < 1) { + cout << "Wrong sensor group format " << group_str << endl; + return ret; + } + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->sensorObj.find (sensorId); + if (idMap != privObj->sensorObj.end()) { + cout << "Duplicate entries for " << group_str << endl; + return ret; + } + + if (configyml[group_str]["enable"]) { + isEnabled = configyml[group_str]["enable"].as(); + if(isEnabled == FALSE) { + ret = true; + goto done; + } + } else { + ret = true; + goto done; + } + + for(YAML::const_iterator itr = configyml[group_str].begin(); + itr != configyml[group_str].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + continue; + } else if (paramKey == "id") { + sensorObj.id = itr->second.as(); + } else if (paramKey == "type") { + sensorObj.type = itr->second.as(); + } else if (paramKey == "description") { + sensorObj.desc = itr->second.as(); + } else if (paramKey == "location") { + std::string str = itr->second.as(); + std::vector vec = split_string (str); + if (vec.size() != 3) { + cout << "Wrong values provided, it should be like lat;lon;alt" << endl; + goto done; + } + for(int i = 0; i < 3; i++) { + sensorObj.location[i] = std::stod(vec[i]); + } + } else if (paramKey == "coordinate") { + std::string str = itr->second.as(); + std::vector vec = split_string (str); + if (vec.size() != 3) { + cout << "Wrong values provided, it should be like x;y;z" << endl; + goto done; + } + for(int i = 0; i < 3; i++) { + sensorObj.coordinate[i] = std::stod(vec[i]); + } + } else { + cout << "Unknown key " << paramKey << " for group [" << group_str << "]\n"; + } + } + privObj->sensorObj.insert (make_pair (sensorId, sensorObj)); + ret = true; + +done: + return ret; +} + +static bool +nvds_msg2p_parse_place_yaml (void *privData, gchar *cfg_file_path, std::string group_str) +{ + bool ret = false; + bool isEnabled = false; + NvDsPayloadPriv *privObj = NULL; + NvDsPlaceObject placeObj; + gint placeId; + + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + if (sscanf (group_str.c_str(), CONFIG_GROUP_PLACE "%u", &placeId) < 1) { + cout << "Wrong place group name " << group_str << endl; + return ret; + } + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->placeObj.find (placeId); + if (idMap != privObj->placeObj.end()) { + cout << "Duplicate entries for " << group_str << endl; + return ret; + } + + if (configyml[group_str]["enable"]) { + isEnabled = configyml[group_str]["enable"].as(); + if(isEnabled == FALSE) { + ret = true; + goto done; + } + } else { + ret = true; + goto done; + } + + for(YAML::const_iterator itr = configyml[group_str].begin(); + itr != configyml[group_str].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + continue; + } else if (paramKey == "id") { + placeObj.id = itr->second.as(); + } else if (paramKey == "type") { + placeObj.type = itr->second.as(); + } else if (paramKey == "name") { + placeObj.name = itr->second.as(); + } else if (paramKey == "location") { + std::string str = itr->second.as(); + std::vector vec = split_string (str); + if (vec.size() != 3) { + cout << "Wrong values provided, it should be like lat;lon;alt" << endl; + goto done; + } + for(int i = 0; i < 3; i++) { + placeObj.location[i] = std::stod(vec[i]); + } + } else if (paramKey == "coordinate") { + std::string str = itr->second.as(); + std::vector vec = split_string (str); + if (vec.size() != 3) { + cout << "Wrong values provided, it should be like x;y;z" << endl; + goto done; + } + for(int i = 0; i < 3; i++) { + placeObj.coordinate[i] = std::stod(vec[i]); + } + } else if (paramKey == "place-sub-field1") { + placeObj.subObj.field1 = itr->second.as(); + } else if (paramKey == "place-sub-field2") { + placeObj.subObj.field2 = itr->second.as(); + } else if (paramKey == "place-sub-field3") { + placeObj.subObj.field3 = itr->second.as(); + } else { + cout << "Unknown key " << paramKey << " for group [" << group_str <<"]\n"; + } + } + privObj->placeObj.insert (pair (placeId, placeObj)); + ret = true; + +done: + return ret; +} + +static bool +nvds_msg2p_parse_analytics_yaml (void *privData, gchar *cfg_file_path, std::string group_str) +{ + bool ret = false; + bool isEnabled = false; + NvDsPayloadPriv *privObj = NULL; + NvDsAnalyticsObject analyticsObj; + gint moduleId; + + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + if (sscanf (group_str.c_str(), CONFIG_GROUP_ANALYTICS "%u", &moduleId) < 1) { + cout << "Wrong analytics module group name " << group_str << endl; + return ret; + } + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->analyticsObj.find (moduleId); + if (idMap != privObj->analyticsObj.end()) { + cout << "Duplicate entries for " << group_str << endl; + return ret; + } + + if (configyml[group_str]["enable"]) { + isEnabled = configyml[group_str]["enable"].as(); + if(isEnabled == FALSE) { + ret = true; + goto done; + } + } else { + ret = true; + goto done; + } + + for(YAML::const_iterator itr = configyml[group_str].begin(); + itr != configyml[group_str].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + continue; + } else if (paramKey == "id") { + analyticsObj.id = itr->second.as(); + } else if (paramKey == "source") { + analyticsObj.source = itr->second.as(); + } else if (paramKey == "description") { + analyticsObj.desc = itr->second.as(); + } else if (paramKey == "version") { + analyticsObj.version = itr->second.as(); + } else { + cout << "Unknown key " << paramKey << " for group [" << group_str <<"]\n"; + } + } + privObj->analyticsObj.insert (make_pair (moduleId, analyticsObj)); + ret = true; + +done: + return ret; +} + +bool nvds_msg2p_parse_csv (void *privData, const gchar *file) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsAnalyticsObject analyticsObj; + NvDsSensorObject sensorObj; + NvDsPlaceObject placeObj; + bool retVal = true; + bool firstRow = true; + string line; + gint i, index = 0; + + ifstream inputFile (file); + if (!inputFile.is_open()) { + cout << "Couldn't open CSV file " << file << endl; + return false; + } + + privObj = (NvDsPayloadPriv *) privData; + + try { + + while (getline (inputFile, line)) { + + if (firstRow) { + // Discard first row as it will have header fields. + firstRow = false; + continue; + } + + vector tokens; + get_csv_tokens (line, tokens); + // Ignore first cameraId field. + i = 1; + + // sensor object fields + sensorObj.id = tokens.at(i++); + sensorObj.type = "Camera"; + sensorObj.desc = tokens.at(i++); + + //Hard coded values but can be read from CSV file. + sensorObj.location[0] = 0; //atof (tokens.at(i++).c_str ()); + sensorObj.location[1] = 0; + sensorObj.location[2] = 0; + sensorObj.coordinate[0] = 0; + sensorObj.coordinate[1] = 0; + sensorObj.coordinate[2] = 0; + + // place object fields + placeObj.id = "Id"; + placeObj.type = "building/garage"; + placeObj.name = "endeavor"; + placeObj.location[0] = 0; + placeObj.location[1] = 0; + placeObj.location[2] = 0; + placeObj.coordinate[0] = 0; + placeObj.coordinate[1] = 0; + placeObj.coordinate[2] = 0; + //Ignore cameraIDstring + i++; + placeObj.subObj.field1 = tokens.at(i++); + placeObj.subObj.field2 = tokens.at(i++); + placeObj.subObj.field3 = tokens.at(i++); + + // analytics object fields + // hard coded values but can be read from CSV file. + analyticsObj.id = ""; + analyticsObj.source = ""; + analyticsObj.desc = ""; + analyticsObj.version = "1.0"; + + privObj->sensorObj.insert (make_pair (index, sensorObj)); + privObj->placeObj.insert (make_pair (index, placeObj)); + privObj->analyticsObj.insert (make_pair (index, analyticsObj)); + + index++; + } + } catch (const std::out_of_range& oor) { + std::cerr << "Out of Range error: " << oor.what() << '\n'; + retVal = false; + } + + inputFile.close (); + return retVal; +} + +bool nvds_msg2p_parse_yaml (void *privData, const gchar *file) +{ + bool retVal = true; + YAML::Node configyml = YAML::LoadFile(file); + std::string sensor_str = "sensor"; + std::string place_str = "place"; + std::string analytics_str = "analytics"; + gchar *cfg_file = (gchar *) malloc(sizeof(gchar *)); + cfg_file = (gchar *) file; + + for(YAML::const_iterator itr = configyml.begin(); itr != configyml.end(); ++itr) + { + std::string paramKey = itr->first.as(); + + if (paramKey.compare(0, sensor_str.size(), sensor_str) == 0) { + retVal = nvds_msg2p_parse_sensor_yaml (privData, cfg_file, paramKey); + } else if (paramKey.compare(0, place_str.size(), place_str) == 0) { + retVal = nvds_msg2p_parse_place_yaml (privData, cfg_file, paramKey); + } else if (paramKey.compare(0, analytics_str.size(), analytics_str) == 0) { + retVal = nvds_msg2p_parse_analytics_yaml (privData, cfg_file, paramKey); + } else { + cout << "Unknown group " << paramKey << endl; + } + + if (!retVal) { + cout << "Failed to parse group " << paramKey << endl; + goto done; + } + } + +done: + return retVal; +} + +bool nvds_msg2p_parse_key_value (void *privData, const gchar *file) +{ + bool retVal = true; + GKeyFile *cfgFile = NULL; + GError *error = NULL; + gchar **groups = NULL; + gchar **group; + + cfgFile = g_key_file_new (); + if (!g_key_file_load_from_file (cfgFile, file, G_KEY_FILE_NONE, &error)) { + g_message ("Failed to load file: %s", error->message); + retVal = false; + goto done; + } + + groups = g_key_file_get_groups (cfgFile, NULL); + + for (group = groups; *group; group++) { + if (!strncmp (*group, CONFIG_GROUP_SENSOR, strlen (CONFIG_GROUP_SENSOR))) { + retVal = nvds_msg2p_parse_sensor (privData, cfgFile, *group); + } else if (!strncmp (*group, CONFIG_GROUP_PLACE, strlen (CONFIG_GROUP_PLACE))) { + retVal = nvds_msg2p_parse_place (privData, cfgFile, *group); + } else if (!strncmp (*group, CONFIG_GROUP_ANALYTICS, strlen (CONFIG_GROUP_ANALYTICS))) { + retVal = nvds_msg2p_parse_analytics (privData, cfgFile, *group); + } else { + cout << "Unknown group " << *group << endl; + } + + if (!retVal) { + cout << "Failed to parse group " << *group << endl; + goto done; + } + } + +done: + if (groups) + g_strfreev (groups); + + if (cfgFile) + g_key_file_free (cfgFile); + + return retVal; +} + +void *create_deepstream_schema_ctx() { + return (void *) new NvDsPayloadPriv; +} + +void destroy_deepstream_schema_ctx(void *ptr) { + NvDsPayloadPriv *privObj = (NvDsPayloadPriv *) ptr; + delete privObj; +} diff --git a/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/deepstream_schema.h b/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/deepstream_schema.h new file mode 100755 index 0000000..e537a24 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/deepstream_schema.h @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** + * @file + * NVIDIA DeepStream: Message Schema payload Generation + * + * @b Description: This file specifies the functions used to generate payload + * based on NVIDIA Deepstream message schema either using eventMsg metadata + * or the NvDSFrame(obj) metadata + */ + +#ifndef NVEVENTMSGCONV_H_ +#define NVEVENTMSGCONV_H_ + +#include "nvdsmeta.h" +#include "nvdsmeta_schema.h" +#include +#include +#include + +using namespace std; + +#define CONFIG_GROUP_SENSOR "sensor" +#define CONFIG_GROUP_PLACE "place" +#define CONFIG_GROUP_ANALYTICS "analytics" + +#define CONFIG_KEY_COORDINATE "coordinate" +#define CONFIG_KEY_DESCRIPTION "description" +#define CONFIG_KEY_ENABLE "enable" +#define CONFIG_KEY_ID "id" +#define CONFIG_KEY_LANE "lane" +#define CONFIG_KEY_LEVEL "level" +#define CONFIG_KEY_LOCATION "location" +#define CONFIG_KEY_NAME "name" +#define CONFIG_KEY_SOURCE "source" +#define CONFIG_KEY_TYPE "type" +#define CONFIG_KEY_VERSION "version" + + +#define CONFIG_KEY_PLACE_SUB_FIELD1 "place-sub-field1" +#define CONFIG_KEY_PLACE_SUB_FIELD2 "place-sub-field2" +#define CONFIG_KEY_PLACE_SUB_FIELD3 "place-sub-field3" + +#define DEFAULT_CSV_FIELDS 10 + + +#define CHECK_ERROR(error) \ + if (error) { \ + cout << "Error: " << error->message << endl; \ + goto done; \ + } + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * Store data parsed from the config file in these structures + */ +struct NvDsPlaceSubObject { + string field1; + string field2; + string field3; +}; + +struct NvDsSensorObject { + string id; + string type; + string desc; + gdouble location[3]; + gdouble coordinate[3]; +}; + +struct NvDsPlaceObject { + string id; + string name; + string type; + gdouble location[3]; + gdouble coordinate[3]; + NvDsPlaceSubObject subObj; +}; + +struct NvDsAnalyticsObject { + string id; + string desc; + string source; + string version; +}; + +struct NvDsPayloadPriv { + unordered_map sensorObj; + unordered_map placeObj; + unordered_map analyticsObj; +}; + +gchar* generate_event_message (void *privData, NvDsEventMsgMeta *meta); +gchar* generate_event_message_minimal (void *privData, NvDsEvent *events, guint size); +gchar* generate_dsmeta_message (void *privData, void *frameMeta, void *objMeta); +gchar* generate_dsmeta_message_minimal (void *privData, void *frameMeta); +void *create_deepstream_schema_ctx(); +void destroy_deepstream_schema_ctx(void *privData); +bool nvds_msg2p_parse_key_value (void *privData, const gchar *file); +bool nvds_msg2p_parse_csv (void *privData, const gchar *file); +bool nvds_msg2p_parse_yaml (void *privData, const gchar *file); + +#ifdef __cplusplus +} +#endif +#endif /* NVEVENTMSGCONV_H_ */ + diff --git a/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/dsmeta_payload.cpp b/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/dsmeta_payload.cpp new file mode 100755 index 0000000..78a69f7 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/dsmeta_payload.cpp @@ -0,0 +1,502 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include "deepstream_schema.h" + +using namespace std; + +#define MAX_TIME_STAMP_LEN (64) + +static void +generate_ts_rfc3339 (char *buf, int buf_size) +{ + time_t tloc; + struct tm tm_log; + struct timespec ts; + char strmsec[6]; //.nnnZ\0 + + clock_gettime (CLOCK_REALTIME, &ts); + memcpy (&tloc, (void *) (&ts.tv_sec), sizeof (time_t)); + gmtime_r (&tloc, &tm_log); + strftime (buf, buf_size, "%Y-%m-%dT%H:%M:%S", &tm_log); + int ms = ts.tv_nsec / 1000000; + g_snprintf (strmsec, sizeof (strmsec), ".%.3dZ", ms); + strncat (buf, strmsec, buf_size); +} + +static JsonObject* generate_place_object (void *privData, NvDsFrameMeta *frame_meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsPlaceObject *dsPlaceObj = NULL; + JsonObject *placeObj; + JsonObject *jobject; + JsonObject *jobject2; + + privObj = (NvDsPayloadPriv *) privData; + auto idMap = privObj->placeObj.find (frame_meta->source_id); + + if (idMap != privObj->placeObj.end()) { + dsPlaceObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_PLACE << frame_meta->source_id + << " in configuration file" << endl; + return NULL; + } + + /* place object + * "place": + { + "id": "string", + "name": "endeavor", + “type”: “garage”, + "location": { + "lat": 30.333, + "lon": -40.555, + "alt": 100.00 + }, + "entrance/aisle": { + "name": "walsh", + "lane": "lane1", + "level": "P2", + "coordinate": { + "x": 1.0, + "y": 2.0, + "z": 3.0 + } + } + } + */ + + placeObj = json_object_new (); + json_object_set_string_member (placeObj, "id", dsPlaceObj->id.c_str()); + json_object_set_string_member (placeObj, "name", dsPlaceObj->name.c_str()); + json_object_set_string_member (placeObj, "type", dsPlaceObj->type.c_str()); + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", dsPlaceObj->location[0]); + json_object_set_double_member (jobject, "lon", dsPlaceObj->location[1]); + json_object_set_double_member (jobject, "alt", dsPlaceObj->location[2]); + json_object_set_object_member (placeObj, "location", jobject); + + // place sub object (user to provide the name for sub place ex: parkingSpot/aisle/entrance..etc + jobject = json_object_new (); + + json_object_set_string_member (jobject, "id", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "name", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "place-sub-field", jobject); + + // coordinates for place sub object + jobject2 = json_object_new (); + json_object_set_double_member (jobject2, "x", dsPlaceObj->coordinate[0]); + json_object_set_double_member (jobject2, "y", dsPlaceObj->coordinate[1]); + json_object_set_double_member (jobject2, "z", dsPlaceObj->coordinate[2]); + json_object_set_object_member (jobject, "coordinate", jobject2); + + return placeObj; +} + +static JsonObject* generate_sensor_object (void *privData, NvDsFrameMeta *frame_meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject *dsSensorObj = NULL; + JsonObject *sensorObj; + JsonObject *jobject; + + privObj = (NvDsPayloadPriv *) privData; + auto idMap = privObj->sensorObj.find (frame_meta->source_id); + + if (idMap != privObj->sensorObj.end()) { + dsSensorObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_SENSOR << frame_meta->source_id + << " in configuration file" << endl; + return NULL; + } + + /* sensor object + * "sensor": { + "id": "string", + "type": "Camera/Puck", + "location": { + "lat": 45.99, + "lon": 35.54, + "alt": 79.03 + }, + "coordinate": { + "x": 5.2, + "y": 10.1, + "z": 11.2 + }, + "description": "Entrance of Endeavor Garage Right Lane" + } + */ + + // sensor object + sensorObj = json_object_new (); + json_object_set_string_member (sensorObj, "id", dsSensorObj->id.c_str()); + json_object_set_string_member (sensorObj, "type", dsSensorObj->type.c_str()); + json_object_set_string_member (sensorObj, "description", dsSensorObj->desc.c_str()); + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", dsSensorObj->location[0]); + json_object_set_double_member (jobject, "lon", dsSensorObj->location[1]); + json_object_set_double_member (jobject, "alt", dsSensorObj->location[2]); + json_object_set_object_member (sensorObj, "location", jobject); + + // coordinate sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "x", dsSensorObj->coordinate[0]); + json_object_set_double_member (jobject, "y", dsSensorObj->coordinate[1]); + json_object_set_double_member (jobject, "z", dsSensorObj->coordinate[2]); + json_object_set_object_member (sensorObj, "coordinate", jobject); + + return sensorObj; +} + +static JsonObject* generate_analytics_module_object (void *privData, NvDsFrameMeta *frame_meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsAnalyticsObject *dsObj = NULL; + JsonObject *analyticsObj; + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->analyticsObj.find (frame_meta->source_id); + + if (idMap != privObj->analyticsObj.end()) { + dsObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_ANALYTICS << frame_meta->source_id + << " in configuration file" << endl; + return NULL; + } + + /* analytics object + * "analyticsModule": { + "id": "string", + "description": "Vehicle Detection and License Plate Recognition", + "confidence": 97.79, + "source": "OpenALR", + "version": "string" + } + */ + + // analytics object + analyticsObj = json_object_new (); + json_object_set_string_member (analyticsObj, "id", dsObj->id.c_str()); + json_object_set_string_member (analyticsObj, "description", dsObj->desc.c_str()); + json_object_set_string_member (analyticsObj, "source", dsObj->source.c_str()); + json_object_set_string_member (analyticsObj, "version", dsObj->version.c_str()); + + return analyticsObj; +} + +static JsonObject* +generate_object_object (void *privData, NvDsFrameMeta *frame_meta, NvDsObjectMeta *obj_meta) +{ + JsonObject *objectObj; + JsonObject *jobject; + gchar tracking_id[64]; + //GList *objectMask = NULL; + + // object object + objectObj = json_object_new (); + if (snprintf (tracking_id, sizeof(tracking_id), "%lu", obj_meta->object_id) + >= (int) sizeof(tracking_id)) + g_warning("Not enough space to copy trackingId"); + json_object_set_string_member (objectObj, "id", tracking_id); + json_object_set_double_member (objectObj, "speed", 0); + json_object_set_double_member (objectObj, "direction", 0); + json_object_set_double_member (objectObj, "orientation", 0); + + jobject = json_object_new (); + json_object_set_double_member (jobject, "confidence", obj_meta->confidence); + + //Fetch object classifiers detected + for(NvDsClassifierMetaList *cl = obj_meta->classifier_meta_list; cl ; cl=cl->next) { + NvDsClassifierMeta *cl_meta = (NvDsClassifierMeta*) cl->data; + + for(NvDsLabelInfoList *ll = cl_meta->label_info_list; ll ; ll=ll->next) { + NvDsLabelInfo *ll_meta = (NvDsLabelInfo*) ll->data; + if(cl_meta->classifier_type != NULL && strcmp("", cl_meta->classifier_type)) + json_object_set_string_member (jobject, cl_meta->classifier_type, ll_meta->result_label); + } + } + json_object_set_object_member (objectObj, obj_meta->obj_label , jobject); + + // bbox sub object + float scaleW = (float) frame_meta->source_frame_width / + (frame_meta->pipeline_width == 0) ? 1:frame_meta->pipeline_width; + float scaleH = (float) frame_meta->source_frame_height / + (frame_meta->pipeline_height == 0) ? 1:frame_meta->pipeline_height; + + float left = obj_meta->rect_params.left * scaleW; + float top = obj_meta->rect_params.top * scaleH; + float width = obj_meta->rect_params.width * scaleW; + float height = obj_meta->rect_params.height * scaleH; + + jobject = json_object_new (); + json_object_set_int_member (jobject, "topleftx", left); + json_object_set_int_member (jobject, "toplefty", top); + json_object_set_int_member (jobject, "bottomrightx", left + width); + json_object_set_int_member (jobject, "bottomrighty", top + height); + json_object_set_object_member (objectObj, "bbox", jobject); + + // location sub object + jobject = json_object_new (); + json_object_set_object_member (objectObj, "location", jobject); + + // coordinate sub object + jobject = json_object_new (); + json_object_set_object_member (objectObj, "coordinate", jobject); + + return objectObj; +} + +static JsonObject* generate_event_object (NvDsObjectMeta *obj_meta) +{ + JsonObject *eventObj; + uuid_t uuid; + gchar uuidStr[37]; + + /* + * "event": { + "id": "event-id", + "type": "entry / exit" + } + */ + + uuid_generate_random (uuid); + uuid_unparse_lower(uuid, uuidStr); + + eventObj = json_object_new (); + json_object_set_string_member (eventObj, "id", uuidStr); + json_object_set_string_member (eventObj, "type", ""); + return eventObj; +} + +gchar* generate_dsmeta_message (void *privData, void *frameMeta, void *objMeta) +{ + JsonNode *rootNode; + JsonObject *rootObj; + JsonObject *placeObj; + JsonObject *sensorObj; + JsonObject *analyticsObj; + JsonObject *eventObj; + JsonObject *objectObj; + gchar *message; + + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)frameMeta; + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)objMeta; + + uuid_t msgId; + gchar msgIdStr[37]; + + uuid_generate_random (msgId); + uuid_unparse_lower(msgId, msgIdStr); + + // place object + placeObj = generate_place_object (privData, frame_meta); + + // sensor object + sensorObj = generate_sensor_object (privData, frame_meta); + + // analytics object + analyticsObj = generate_analytics_module_object (privData, frame_meta); + + // object object + objectObj = generate_object_object (privData, frame_meta, obj_meta); + // event object + eventObj = generate_event_object (obj_meta); + + char ts[MAX_TIME_STAMP_LEN + 1]; + generate_ts_rfc3339 (ts, MAX_TIME_STAMP_LEN); + + // root object + rootObj = json_object_new (); + json_object_set_string_member (rootObj, "messageid", msgIdStr); + json_object_set_string_member (rootObj, "mdsversion", "1.0"); + json_object_set_string_member (rootObj, "@timestamp", ts); + json_object_set_object_member (rootObj, "place", placeObj); + json_object_set_object_member (rootObj, "sensor", sensorObj); + json_object_set_object_member (rootObj, "analyticsModule", analyticsObj); + json_object_set_object_member (rootObj, "object", objectObj); + json_object_set_object_member (rootObj, "event", eventObj); + + json_object_set_string_member (rootObj, "videoPath", ""); + + //Search for any custom message blob within frame usermeta list + JsonArray *jArray = json_array_new (); + for (NvDsUserMetaList *l = frame_meta->frame_user_meta_list; l; l = l->next) { + NvDsUserMeta *frame_usermeta = (NvDsUserMeta *) l->data; + if(frame_usermeta && frame_usermeta->base_meta.meta_type == NVDS_CUSTOM_MSG_BLOB) { + NvDsCustomMsgInfo *custom_blob = (NvDsCustomMsgInfo *) frame_usermeta->user_meta_data; + string msg = string((const char *) custom_blob->message, custom_blob->size); + json_array_add_string_element (jArray, msg.c_str()); + } + } + if(json_array_get_length(jArray) > 0) + json_object_set_array_member (rootObj, "customMessage", jArray); + else + json_array_unref(jArray); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, rootObj); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (rootObj); + + return message; + +} + +gchar* generate_dsmeta_message_minimal (void *privData, void *frameMeta) +{ + /* + The JSON structure of the frame + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + ".......object-1 attributes...........", + ".......object-2 attributes...........", + ".......object-3 attributes..........." + ] + } + */ + + /* + An example object with Vehicle object-type + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + "957|1834|150|1918|215|Vehicle|#|sedan|Bugatti|M|blue|CA 444|California|0.8", + "..........." + ] + } + */ + + JsonNode *rootNode; + JsonObject *jobject; + JsonArray *jArray; + stringstream ss; + gchar *message = NULL; + + jArray = json_array_new (); + + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) frameMeta; + for (NvDsObjectMetaList *obj_l = frame_meta->obj_meta_list; obj_l; obj_l = obj_l->next) { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *) obj_l->data; + if (obj_meta == NULL) { + // Ignore Null object. + continue; + } + + // bbox sub object + float scaleW = (float) frame_meta->source_frame_width / + (frame_meta->pipeline_width == 0) ? 1:frame_meta->pipeline_width; + float scaleH = (float) frame_meta->source_frame_height / + (frame_meta->pipeline_height == 0) ? 1:frame_meta->pipeline_height; + + float left = obj_meta->rect_params.left * scaleW; + float top = obj_meta->rect_params.top * scaleH; + float width = obj_meta->rect_params.width * scaleW; + float height = obj_meta->rect_params.height * scaleH; + + ss.str(""); + ss.clear(); + ss << obj_meta->object_id << "|" << left << "|" << top + << "|" << left + width << "|" << top + height + << "|" << obj_meta->obj_label; + + if(g_list_length(obj_meta->classifier_meta_list) > 0) { + ss << "|#"; + //Add classifiers for the object, if any + for(NvDsClassifierMetaList *cl = obj_meta->classifier_meta_list; cl ; cl=cl->next) { + NvDsClassifierMeta *cl_meta = (NvDsClassifierMeta*) cl->data; + for(NvDsLabelInfoList *ll = cl_meta->label_info_list; ll ; ll=ll->next) { + NvDsLabelInfo *ll_meta = (NvDsLabelInfo*) ll->data; + ss<< "|" << ll_meta->result_label; + } + } + ss << "|" << obj_meta->confidence; + } + json_array_add_string_element (jArray, ss.str().c_str()); + } + + //generate timestamp + char ts[MAX_TIME_STAMP_LEN + 1]; + generate_ts_rfc3339 (ts, MAX_TIME_STAMP_LEN); + + //fetch sensor id + string sensorId="0"; + NvDsPayloadPriv *privObj = (NvDsPayloadPriv *) privData; + auto idMap = privObj->sensorObj.find (frame_meta->source_id); + if (idMap != privObj->sensorObj.end()) { + NvDsSensorObject &obj = privObj->sensorObj[frame_meta->source_id]; + sensorId = obj.id; + } + + jobject = json_object_new (); + json_object_set_string_member (jobject, "version", "4.0"); + json_object_set_string_member (jobject, "id", to_string(frame_meta->frame_num).c_str()); + json_object_set_string_member (jobject, "@timestamp", ts); + json_object_set_string_member (jobject, "sensorId", sensorId.c_str()); + + json_object_set_array_member (jobject, "objects", jArray); + + JsonArray *custMsgjArray = json_array_new (); + //Search for any custom message blob within frame usermeta list + for (NvDsUserMetaList *l = frame_meta->frame_user_meta_list; l; l = l->next) { + NvDsUserMeta *frame_usermeta = (NvDsUserMeta *) l->data; + if(frame_usermeta && frame_usermeta->base_meta.meta_type == NVDS_CUSTOM_MSG_BLOB) { + NvDsCustomMsgInfo *custom_blob = (NvDsCustomMsgInfo *) frame_usermeta->user_meta_data; + string msg = string((const char *) custom_blob->message, custom_blob->size); + json_array_add_string_element (custMsgjArray, msg.c_str()); + } + } + if(json_array_get_length(custMsgjArray) > 0) + json_object_set_array_member (jobject, "customMessage", custMsgjArray); + else + json_array_unref(custMsgjArray); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, jobject); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (jobject); + + return message; +} diff --git a/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/eventmsg_payload.cpp b/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/eventmsg_payload.cpp new file mode 100755 index 0000000..a218ee1 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/eventmsg_payload.cpp @@ -0,0 +1,839 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include "deepstream_schema.h" + + +static JsonObject* +generate_place_object (void *privData, NvDsEventMsgMeta *meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsPlaceObject *dsPlaceObj = NULL; + JsonObject *placeObj; + JsonObject *jobject; + JsonObject *jobject2; + + privObj = (NvDsPayloadPriv *) privData; + auto idMap = privObj->placeObj.find (meta->placeId); + + if (idMap != privObj->placeObj.end()) { + dsPlaceObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_PLACE << meta->placeId + << " in configuration file" << endl; + return NULL; + } + + /* place object + * "place": + { + "id": "string", + "name": "endeavor", + “type”: “garage”, + "location": { + "lat": 30.333, + "lon": -40.555, + "alt": 100.00 + }, + "entrance/aisle": { + "name": "walsh", + "lane": "lane1", + "level": "P2", + "coordinate": { + "x": 1.0, + "y": 2.0, + "z": 3.0 + } + } + } + */ + + placeObj = json_object_new (); + json_object_set_string_member (placeObj, "id", dsPlaceObj->id.c_str()); + json_object_set_string_member (placeObj, "name", dsPlaceObj->name.c_str()); + json_object_set_string_member (placeObj, "type", dsPlaceObj->type.c_str()); + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", dsPlaceObj->location[0]); + json_object_set_double_member (jobject, "lon", dsPlaceObj->location[1]); + json_object_set_double_member (jobject, "alt", dsPlaceObj->location[2]); + json_object_set_object_member (placeObj, "location", jobject); + + // parkingSpot / aisle /entrance sub object + jobject = json_object_new (); + + switch (meta->type) { + case NVDS_EVENT_MOVING: + case NVDS_EVENT_STOPPED: + json_object_set_string_member (jobject, "id", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "name", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "aisle", jobject); + break; + case NVDS_EVENT_EMPTY: + case NVDS_EVENT_PARKED: + json_object_set_string_member (jobject, "id", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "type", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "parkingSpot", jobject); + break; + case NVDS_EVENT_ENTRY: + case NVDS_EVENT_EXIT: + if (meta->objType == NVDS_OBJECT_TYPE_VEHICLE) { + json_object_set_string_member (jobject, "id", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "name", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "aisle", jobject); + } else { + json_object_set_string_member (jobject, "name", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "lane", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "entrance", jobject); + } + break; + default: + cout << "Event type not implemented " << endl; + break; + } + + // coordinate sub sub object + jobject2 = json_object_new (); + json_object_set_double_member (jobject2, "x", dsPlaceObj->coordinate[0]); + json_object_set_double_member (jobject2, "y", dsPlaceObj->coordinate[1]); + json_object_set_double_member (jobject2, "z", dsPlaceObj->coordinate[2]); + json_object_set_object_member (jobject, "coordinate", jobject2); + + return placeObj; +} + +static JsonObject* +generate_sensor_object (void *privData, NvDsEventMsgMeta *meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject *dsSensorObj = NULL; + JsonObject *sensorObj; + JsonObject *jobject; + + privObj = (NvDsPayloadPriv *) privData; + auto idMap = privObj->sensorObj.find (meta->sensorId); + + if (idMap != privObj->sensorObj.end()) { + dsSensorObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_SENSOR << meta->sensorId + << " in configuration file" << endl; + return NULL; + } + + /* sensor object + * "sensor": { + "id": "string", + "type": "Camera/Puck", + "location": { + "lat": 45.99, + "lon": 35.54, + "alt": 79.03 + }, + "coordinate": { + "x": 5.2, + "y": 10.1, + "z": 11.2 + }, + "description": "Entrance of Endeavor Garage Right Lane" + } + */ + + // sensor object + sensorObj = json_object_new (); + json_object_set_string_member (sensorObj, "id", dsSensorObj->id.c_str()); + json_object_set_string_member (sensorObj, "type", dsSensorObj->type.c_str()); + json_object_set_string_member (sensorObj, "description", dsSensorObj->desc.c_str()); + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", dsSensorObj->location[0]); + json_object_set_double_member (jobject, "lon", dsSensorObj->location[1]); + json_object_set_double_member (jobject, "alt", dsSensorObj->location[2]); + json_object_set_object_member (sensorObj, "location", jobject); + + // coordinate sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "x", dsSensorObj->coordinate[0]); + json_object_set_double_member (jobject, "y", dsSensorObj->coordinate[1]); + json_object_set_double_member (jobject, "z", dsSensorObj->coordinate[2]); + json_object_set_object_member (sensorObj, "coordinate", jobject); + + return sensorObj; +} + +static JsonObject* +generate_analytics_module_object (void *privData, NvDsEventMsgMeta *meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsAnalyticsObject *dsObj = NULL; + JsonObject *analyticsObj; + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->analyticsObj.find (meta->moduleId); + + if (idMap != privObj->analyticsObj.end()) { + dsObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_ANALYTICS << meta->moduleId + << " in configuration file" << endl; + return NULL; + } + + /* analytics object + * "analyticsModule": { + "id": "string", + "description": "Vehicle Detection and License Plate Recognition", + "confidence": 97.79, + "source": "OpenALR", + "version": "string" + } + */ + + // analytics object + analyticsObj = json_object_new (); + json_object_set_string_member (analyticsObj, "id", dsObj->id.c_str()); + json_object_set_string_member (analyticsObj, "description", dsObj->desc.c_str()); + json_object_set_string_member (analyticsObj, "source", dsObj->source.c_str()); + json_object_set_string_member (analyticsObj, "version", dsObj->version.c_str()); + + return analyticsObj; +} + +static JsonObject* +generate_event_object (void *privData, NvDsEventMsgMeta *meta) +{ + JsonObject *eventObj; + uuid_t uuid; + gchar uuidStr[37]; + + /* + * "event": { + "id": "event-id", + "type": "entry / exit" + } + */ + + uuid_generate_random (uuid); + uuid_unparse_lower(uuid, uuidStr); + + eventObj = json_object_new (); + json_object_set_string_member (eventObj, "id", uuidStr); + + switch (meta->type) { + case NVDS_EVENT_ENTRY: + json_object_set_string_member (eventObj, "type", "entry"); + break; + case NVDS_EVENT_EXIT: + json_object_set_string_member (eventObj, "type", "exit"); + break; + case NVDS_EVENT_MOVING: + json_object_set_string_member (eventObj, "type", "moving"); + break; + case NVDS_EVENT_STOPPED: + json_object_set_string_member (eventObj, "type", "stopped"); + break; + case NVDS_EVENT_PARKED: + json_object_set_string_member (eventObj, "type", "parked"); + break; + case NVDS_EVENT_EMPTY: + json_object_set_string_member (eventObj, "type", "empty"); + break; + case NVDS_EVENT_RESET: + json_object_set_string_member (eventObj, "type", "reset"); + break; + default: + cout << "Unknown event type " << endl; + break; + } + + return eventObj; +} + +static JsonObject* +generate_object_object (void *privData, NvDsEventMsgMeta *meta) +{ + JsonObject *objectObj; + JsonObject *jobject; + guint i; + gchar tracking_id[64]; + GList *objectMask = NULL; + + // object object + objectObj = json_object_new (); + if (snprintf (tracking_id, sizeof(tracking_id), "%lu", meta->trackingId) + >= (int) sizeof(tracking_id)) + g_warning("Not enough space to copy trackingId"); + json_object_set_string_member (objectObj, "id", tracking_id); + json_object_set_double_member (objectObj, "speed", 0); + json_object_set_double_member (objectObj, "direction", 0); + json_object_set_double_member (objectObj, "orientation", 0); + + switch (meta->objType) { + case NVDS_OBJECT_TYPE_VEHICLE: + // vehicle sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsVehicleObject *dsObj = (NvDsVehicleObject *) meta->extMsg; + if (dsObj) { + json_object_set_string_member (jobject, "type", dsObj->type); + json_object_set_string_member (jobject, "make", dsObj->make); + json_object_set_string_member (jobject, "model", dsObj->model); + json_object_set_string_member (jobject, "color", dsObj->color); + json_object_set_string_member (jobject, "licenseState", dsObj->region); + json_object_set_string_member (jobject, "license", dsObj->license); + json_object_set_double_member (jobject, "confidence", meta->confidence); + } + } else { + // No vehicle object in meta data. Attach empty vehicle sub object. + json_object_set_string_member (jobject, "type", ""); + json_object_set_string_member (jobject, "make", ""); + json_object_set_string_member (jobject, "model", ""); + json_object_set_string_member (jobject, "color", ""); + json_object_set_string_member (jobject, "licenseState", ""); + json_object_set_string_member (jobject, "license", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "vehicle", jobject); + break; + case NVDS_OBJECT_TYPE_PERSON: + // person sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsPersonObject *dsObj = (NvDsPersonObject *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "apparel", dsObj->apparel); + json_object_set_double_member (jobject, "confidence", meta->confidence); + } + } else { + // No person object in meta data. Attach empty person sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "apparel", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "person", jobject); + break; + case NVDS_OBJECT_TYPE_FACE: + // face sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsFaceObject *dsObj = (NvDsFaceObject *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "glasses", dsObj->glasses); + json_object_set_string_member (jobject, "facialhair", dsObj->facialhair); + json_object_set_string_member (jobject, "name", dsObj->name); + json_object_set_string_member (jobject, "eyecolor", dsObj->eyecolor); + json_object_set_double_member (jobject, "confidence", meta->confidence); + } + } else { + // No face object in meta data. Attach empty face sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "glasses", ""); + json_object_set_string_member (jobject, "facialhair", ""); + json_object_set_string_member (jobject, "name", ""); + json_object_set_string_member (jobject, "eyecolor", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "face", jobject); + break; + case NVDS_OBJECT_TYPE_VEHICLE_EXT: + // vehicle sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsVehicleObjectExt *dsObj = (NvDsVehicleObjectExt *) meta->extMsg; + if (dsObj) { + json_object_set_string_member (jobject, "type", dsObj->type); + json_object_set_string_member (jobject, "make", dsObj->make); + json_object_set_string_member (jobject, "model", dsObj->model); + json_object_set_string_member (jobject, "color", dsObj->color); + json_object_set_string_member (jobject, "licenseState", dsObj->region); + json_object_set_string_member (jobject, "license", dsObj->license); + json_object_set_double_member (jobject, "confidence", meta->confidence); + + objectMask = dsObj->mask; + } + } else { + // No vehicle object in meta data. Attach empty vehicle sub object. + json_object_set_string_member (jobject, "type", ""); + json_object_set_string_member (jobject, "make", ""); + json_object_set_string_member (jobject, "model", ""); + json_object_set_string_member (jobject, "color", ""); + json_object_set_string_member (jobject, "licenseState", ""); + json_object_set_string_member (jobject, "license", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "vehicle", jobject); + break; + case NVDS_OBJECT_TYPE_PERSON_EXT: + // person sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsPersonObjectExt *dsObj = (NvDsPersonObjectExt *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "apparel", dsObj->apparel); + json_object_set_double_member (jobject, "confidence", meta->confidence); + + objectMask = dsObj->mask; + } + } else { + // No person object in meta data. Attach empty person sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "apparel", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "person", jobject); + break; + case NVDS_OBJECT_TYPE_FACE_EXT: + // face sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsFaceObjectExt *dsObj = (NvDsFaceObjectExt *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "glasses", dsObj->glasses); + json_object_set_string_member (jobject, "facialhair", dsObj->facialhair); + json_object_set_string_member (jobject, "name", dsObj->name); + json_object_set_string_member (jobject, "eyecolor", dsObj->eyecolor); + json_object_set_double_member (jobject, "confidence", meta->confidence); + + objectMask = dsObj->mask; + } + } else { + // No face object in meta data. Attach empty face sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "glasses", ""); + json_object_set_string_member (jobject, "facialhair", ""); + json_object_set_string_member (jobject, "name", ""); + json_object_set_string_member (jobject, "eyecolor", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "face", jobject); + break; + case NVDS_OBJECT_TYPE_UNKNOWN: + if(!meta->objectId) { + break; + } + /** No information to add; object type unknown within NvDsEventMsgMeta */ + jobject = json_object_new (); + json_object_set_object_member (objectObj, meta->objectId, jobject); + break; + default: + cout << "Object type not implemented" << endl; + } + + // bbox sub object + jobject = json_object_new (); + json_object_set_int_member (jobject, "topleftx", meta->bbox.left); + json_object_set_int_member (jobject, "toplefty", meta->bbox.top); + json_object_set_int_member (jobject, "bottomrightx", meta->bbox.left + meta->bbox.width); + json_object_set_int_member (jobject, "bottomrighty", meta->bbox.top + meta->bbox.height); + json_object_set_object_member (objectObj, "bbox", jobject); + + if (objectMask) { + GList *l; + JsonArray *maskArray = json_array_sized_new (g_list_length(objectMask)); + + for (l = objectMask; l != NULL; l = l->next) { + GArray *polygon = (GArray *) l->data; + JsonArray *polygonArray = json_array_sized_new (polygon->len); + + for (i = 0; i < polygon->len; i++) { + gdouble value = g_array_index (polygon, gdouble, i); + + json_array_add_double_element (polygonArray, value); + } + + json_array_add_array_element (maskArray, polygonArray); + } + + json_object_set_array_member (objectObj, "maskoutline", maskArray); + } + + // signature sub array + if (meta->objSignature.size) { + JsonArray *jArray = json_array_sized_new (meta->objSignature.size); + + for (i = 0; i < meta->objSignature.size; i++) { + json_array_add_double_element (jArray, meta->objSignature.signature[i]); + } + json_object_set_array_member (objectObj, "signature", jArray); + } + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", meta->location.lat); + json_object_set_double_member (jobject, "lon", meta->location.lon); + json_object_set_double_member (jobject, "alt", meta->location.alt); + json_object_set_object_member (objectObj, "location", jobject); + + // coordinate sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "x", meta->coordinate.x); + json_object_set_double_member (jobject, "y", meta->coordinate.y); + json_object_set_double_member (jobject, "z", meta->coordinate.z); + json_object_set_object_member (objectObj, "coordinate", jobject); + + return objectObj; +} + +gchar* generate_event_message (void *privData, NvDsEventMsgMeta *meta) +{ + JsonNode *rootNode; + JsonObject *rootObj; + JsonObject *placeObj; + JsonObject *sensorObj; + JsonObject *analyticsObj; + JsonObject *eventObj; + JsonObject *objectObj; + gchar *message; + + uuid_t msgId; + gchar msgIdStr[37]; + + uuid_generate_random (msgId); + uuid_unparse_lower(msgId, msgIdStr); + + // place object + placeObj = generate_place_object (privData, meta); + + // sensor object + sensorObj = generate_sensor_object (privData, meta); + + // analytics object + analyticsObj = generate_analytics_module_object (privData, meta); + + // object object + objectObj = generate_object_object (privData, meta); + + // event object + eventObj = generate_event_object (privData, meta); + + // root object + rootObj = json_object_new (); + json_object_set_string_member (rootObj, "messageid", msgIdStr); + json_object_set_string_member (rootObj, "mdsversion", "1.0"); + json_object_set_string_member (rootObj, "@timestamp", meta->ts); + json_object_set_object_member (rootObj, "place", placeObj); + json_object_set_object_member (rootObj, "sensor", sensorObj); + json_object_set_object_member (rootObj, "analyticsModule", analyticsObj); + json_object_set_object_member (rootObj, "object", objectObj); + json_object_set_object_member (rootObj, "event", eventObj); + + if (meta->videoPath) + json_object_set_string_member (rootObj, "videoPath", meta->videoPath); + else + json_object_set_string_member (rootObj, "videoPath", ""); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, rootObj); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (rootObj); + + return message; +} + +static const gchar* +object_enum_to_str (NvDsObjectType type, gchar* objectId) +{ + switch (type) { + case NVDS_OBJECT_TYPE_VEHICLE: + return "Vehicle"; + case NVDS_OBJECT_TYPE_FACE: + return "Face"; + case NVDS_OBJECT_TYPE_PERSON: + return "Person"; + case NVDS_OBJECT_TYPE_BAG: + return "Bag"; + case NVDS_OBJECT_TYPE_BICYCLE: + return "Bicycle"; + case NVDS_OBJECT_TYPE_ROADSIGN: + return "RoadSign"; + case NVDS_OBJECT_TYPE_CUSTOM: + return "Custom"; + case NVDS_OBJECT_TYPE_UNKNOWN: + return objectId ? objectId : "Unknown"; + default: + return "Unknown"; + } +} + +static const gchar* +to_str (gchar* cstr) +{ + return reinterpret_cast(cstr) ? cstr : ""; +} + +static const gchar * +sensor_id_to_str (void *privData, gint sensorId) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject *dsObj = NULL; + + g_return_val_if_fail (privData, NULL); + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->sensorObj.find (sensorId); + if (idMap != privObj->sensorObj.end()) { + dsObj = &idMap->second; + return dsObj->id.c_str(); + } else { + cout << "No entry for " CONFIG_GROUP_SENSOR << sensorId + << " in configuration file" << endl; + return NULL; + } +} + +static void +generate_mask_array (NvDsEventMsgMeta *meta, JsonArray *jArray, GList *mask) +{ + unsigned int i; + GList *l; + stringstream ss; + bool started = false; + + ss << meta->trackingId << "|" << g_list_length(mask); + + for (l = mask; l != NULL; l = l->next) { + GArray *polygon = (GArray *) l->data; + + if (started) + ss << "|#"; + + started = true; + + for (i = 0; i < polygon->len; i++) { + gdouble value = g_array_index (polygon, gdouble, i); + ss << "|" << value; + } + } + json_array_add_string_element (jArray, ss.str().c_str()); +} + +gchar* generate_event_message_minimal (void *privData, NvDsEvent *events, guint size) +{ + /* + The JSON structure of the frame + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + ".......object-1 attributes...........", + ".......object-2 attributes...........", + ".......object-3 attributes..........." + ] + } + */ + + /* + An example object with Vehicle object-type + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + "957|1834|150|1918|215|Vehicle|#|sedan|Bugatti|M|blue|CA 444|California|0.8", + "..........." + ] + } + */ + + JsonNode *rootNode; + JsonObject *jobject; + JsonArray *jArray; + JsonArray *maskArray = NULL; + guint i; + stringstream ss; + gchar *message = NULL; + + jArray = json_array_new (); + + for (i = 0; i < size; i++) { + GList *objectMask = NULL; + + ss.str(""); + ss.clear(); + + NvDsEventMsgMeta *meta = events[i].metadata; + ss << meta->trackingId << "|" << meta->bbox.left << "|" << meta->bbox.top + << "|" << meta->bbox.left + meta->bbox.width << "|" << meta->bbox.top + meta->bbox.height + << "|" << object_enum_to_str (meta->objType, meta->objectId); + + if (meta->extMsg && meta->extMsgSize) { + // Attach secondary inference attributes. + switch (meta->objType) { + case NVDS_OBJECT_TYPE_VEHICLE: { + NvDsVehicleObject *dsObj = (NvDsVehicleObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->type) << "|" << to_str(dsObj->make) << "|" + << to_str(dsObj->model) << "|" << to_str(dsObj->color) << "|" << to_str(dsObj->license) + << "|" << to_str(dsObj->region) << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_PERSON: { + NvDsPersonObject *dsObj = (NvDsPersonObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->apparel) + << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_FACE: { + NvDsFaceObject *dsObj = (NvDsFaceObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->glasses) + << "|" << to_str(dsObj->facialhair) << "|" << to_str(dsObj->name) << "|" + << "|" << to_str(dsObj->eyecolor) << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_VEHICLE_EXT: { + NvDsVehicleObjectExt *dsObj = (NvDsVehicleObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->type) << "|" << to_str(dsObj->make) << "|" + << to_str(dsObj->model) << "|" << to_str(dsObj->color) << "|" << to_str(dsObj->license) + << "|" << to_str(dsObj->region) << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + case NVDS_OBJECT_TYPE_PERSON_EXT: { + NvDsPersonObjectExt *dsObj = (NvDsPersonObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->apparel) + << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + case NVDS_OBJECT_TYPE_FACE_EXT: { + NvDsFaceObjectExt *dsObj = (NvDsFaceObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->glasses) + << "|" << to_str(dsObj->facialhair) << "|" << to_str(dsObj->name) << "|" + << "|" << to_str(dsObj->eyecolor) << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + default: + cout << "Object type (" << meta->objType << ") not implemented" << endl; + break; + } + } + + if (objectMask) { + if (maskArray == NULL) + maskArray = json_array_new (); + generate_mask_array (meta, maskArray, objectMask); + } + + json_array_add_string_element (jArray, ss.str().c_str()); + } + + // It is assumed that all events / objects are associated with same frame. + // Therefore ts / sensorId / frameId of first object can be used. + + jobject = json_object_new (); + json_object_set_string_member (jobject, "version", "4.0"); + json_object_set_string_member (jobject, "id", to_string(events[0].metadata->frameId).c_str()); + json_object_set_string_member (jobject, "@timestamp", events[0].metadata->ts); + if (events[0].metadata->sensorStr) { + json_object_set_string_member (jobject, "sensorId", events[0].metadata->sensorStr); + } else if ((NvDsPayloadPriv *) privData) { + json_object_set_string_member (jobject, "sensorId", + to_str((gchar *) sensor_id_to_str (privData, events[0].metadata->sensorId))); + } else { + json_object_set_string_member (jobject, "sensorId", "0"); + } + + json_object_set_array_member (jobject, "objects", jArray); + if (maskArray && json_array_get_length (maskArray) > 0) + json_object_set_array_member (jobject, "masks", maskArray); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, jobject); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (jobject); + + return message; +} diff --git a/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/eventmsg_payload_peoplenet.cpp b/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/eventmsg_payload_peoplenet.cpp new file mode 100755 index 0000000..8f948e0 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/eventmsg_payload_peoplenet.cpp @@ -0,0 +1,423 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include "deepstream_schema.h" + +static JsonObject* +generate_event_object (void *privData, NvDsEventMsgMeta *meta) +{ + JsonObject *eventObj; + uuid_t uuid; + gchar uuidStr[37]; + + /* + * "event": { + "id": "event-id", + "type": "entry / exit" + } + */ + + uuid_generate_random (uuid); + uuid_unparse_lower(uuid, uuidStr); + + eventObj = json_object_new (); + json_object_set_string_member (eventObj, "id", uuidStr); + + switch (meta->type) { + case NVDS_EVENT_ENTRY: + json_object_set_string_member (eventObj, "type", "entry"); + break; + case NVDS_EVENT_EXIT: + json_object_set_string_member (eventObj, "type", "exit"); + break; + case NVDS_EVENT_MOVING: + json_object_set_string_member (eventObj, "type", "moving"); + break; + case NVDS_EVENT_STOPPED: + json_object_set_string_member (eventObj, "type", "stopped"); + break; + case NVDS_EVENT_PARKED: + json_object_set_string_member (eventObj, "type", "parked"); + break; + case NVDS_EVENT_EMPTY: + json_object_set_string_member (eventObj, "type", "empty"); + break; + case NVDS_EVENT_RESET: + json_object_set_string_member (eventObj, "type", "reset"); + break; + default: + cout << "Unknown event type " << endl; + break; + } + + return eventObj; +} + +static JsonObject* +generate_object_object (void *privData, NvDsEventMsgMeta *meta) +{ + JsonObject *objectObj; + JsonObject *jobject; + guint i; + gchar tracking_id[64]; + GList *objectMask = NULL; + + // object object + objectObj = json_object_new (); + if (snprintf (tracking_id, sizeof(tracking_id), "%lu", meta->trackingId) + >= (int) sizeof(tracking_id)) + g_warning("Not enough space to copy trackingId"); + json_object_set_string_member (objectObj, "id", tracking_id); + json_object_set_double_member (objectObj, "speed", 0); + json_object_set_double_member (objectObj, "direction", 0); + json_object_set_double_member (objectObj, "orientation", 0); + + switch (meta->objType) { + case NVDS_OBJECT_TYPE_PERSON: + // person sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsPersonObject *dsObj = (NvDsPersonObject *) meta->extMsg; + if (dsObj) { + json_object_set_string_member (jobject, "hasBasket", dsObj->hasBasket); + json_object_set_double_member (jobject, "confidence", meta->confidence); + } + } else { + // No person object in meta data. Attach empty person sub object. + json_object_set_string_member (jobject, "hasBasket", "NoBasket"); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_string_member (objectObj, "detection", "person"); + json_object_set_object_member (objectObj, "obj_prop", jobject); + break; + case NVDS_OBJECT_TYPE_UNKNOWN: + if(!meta->objectId) { + break; + } + /** No information to add; object type unknown within NvDsEventMsgMeta */ + jobject = json_object_new (); + json_object_set_object_member (objectObj, meta->objectId, jobject); + break; + default: + cout << "Object type not implemented" << endl; + } + + // bbox sub object + jobject = json_object_new (); + json_object_set_int_member (jobject, "topleftx", meta->bbox.left); + json_object_set_int_member (jobject, "toplefty", meta->bbox.top); + json_object_set_int_member (jobject, "bottomrightx", meta->bbox.left + meta->bbox.width); + json_object_set_int_member (jobject, "bottomrighty", meta->bbox.top + meta->bbox.height); + json_object_set_object_member (objectObj, "bbox", jobject); + + return objectObj; +} + +gchar* generate_event_message (void *privData, NvDsEventMsgMeta *meta) +{ + JsonNode *rootNode; + JsonObject *rootObj; + JsonObject *eventObj; + JsonObject *objectObj; + gchar *message; + + uuid_t msgId; + gchar msgIdStr[37]; + + uuid_generate_random (msgId); + uuid_unparse_lower(msgId, msgIdStr); + + // object object + objectObj = generate_object_object (privData, meta); + + // event object + eventObj = generate_event_object (privData, meta); + + // root object + rootObj = json_object_new (); + json_object_set_string_member (rootObj, "messageid", msgIdStr); + json_object_set_string_member (rootObj, "mdsversion", "1.0"); + json_object_set_string_member (rootObj, "timestamp", meta->ts); + json_object_set_object_member (rootObj, "object", objectObj); + json_object_set_object_member (rootObj, "event_des", eventObj); + + if (meta->videoPath) + json_object_set_string_member (rootObj, "videoPath", meta->videoPath); + else + json_object_set_string_member (rootObj, "videoPath", ""); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, rootObj); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (rootObj); + + return message; +} + +static const gchar* +object_enum_to_str (NvDsObjectType type, gchar* objectId) +{ + switch (type) { + case NVDS_OBJECT_TYPE_VEHICLE: + return "Vehicle"; + case NVDS_OBJECT_TYPE_FACE: + return "Face"; + case NVDS_OBJECT_TYPE_PERSON: + return "Person"; + case NVDS_OBJECT_TYPE_BAG: + return "Bag"; + case NVDS_OBJECT_TYPE_BICYCLE: + return "Bicycle"; + case NVDS_OBJECT_TYPE_ROADSIGN: + return "RoadSign"; + case NVDS_OBJECT_TYPE_CUSTOM: + return "Custom"; + case NVDS_OBJECT_TYPE_UNKNOWN: + return objectId ? objectId : "Unknown"; + default: + return "Unknown"; + } +} + +static const gchar* +to_str (gchar* cstr) +{ + return reinterpret_cast(cstr) ? cstr : ""; +} + +static const gchar * +sensor_id_to_str (void *privData, gint sensorId) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject *dsObj = NULL; + + g_return_val_if_fail (privData, NULL); + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->sensorObj.find (sensorId); + if (idMap != privObj->sensorObj.end()) { + dsObj = &idMap->second; + return dsObj->id.c_str(); + } else { + cout << "No entry for " CONFIG_GROUP_SENSOR << sensorId + << " in configuration file" << endl; + return NULL; + } +} + +static void +generate_mask_array (NvDsEventMsgMeta *meta, JsonArray *jArray, GList *mask) +{ + unsigned int i; + GList *l; + stringstream ss; + bool started = false; + + ss << meta->trackingId << "|" << g_list_length(mask); + + for (l = mask; l != NULL; l = l->next) { + GArray *polygon = (GArray *) l->data; + + if (started) + ss << "|#"; + + started = true; + + for (i = 0; i < polygon->len; i++) { + gdouble value = g_array_index (polygon, gdouble, i); + ss << "|" << value; + } + } + json_array_add_string_element (jArray, ss.str().c_str()); +} + +gchar* generate_event_message_minimal (void *privData, NvDsEvent *events, guint size) +{ + /* + The JSON structure of the frame + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + ".......object-1 attributes...........", + ".......object-2 attributes...........", + ".......object-3 attributes..........." + ] + } + */ + + /* + An example object with Vehicle object-type + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + "957|1834|150|1918|215|Vehicle|#|sedan|Bugatti|M|blue|CA 444|California|0.8", + "..........." + ] + } + */ + + JsonNode *rootNode; + JsonObject *jobject; + JsonArray *jArray; + JsonArray *maskArray = NULL; + guint i; + stringstream ss; + gchar *message = NULL; + + jArray = json_array_new (); + + for (i = 0; i < size; i++) { + GList *objectMask = NULL; + + ss.str(""); + ss.clear(); + + NvDsEventMsgMeta *meta = events[i].metadata; + ss << meta->trackingId << "|" << meta->bbox.left << "|" << meta->bbox.top + << "|" << meta->bbox.left + meta->bbox.width << "|" << meta->bbox.top + meta->bbox.height + << "|" << object_enum_to_str (meta->objType, meta->objectId); + + if (meta->extMsg && meta->extMsgSize) { + // Attach secondary inference attributes. + switch (meta->objType) { + case NVDS_OBJECT_TYPE_VEHICLE: { + NvDsVehicleObject *dsObj = (NvDsVehicleObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->type) << "|" << to_str(dsObj->make) << "|" + << to_str(dsObj->model) << "|" << to_str(dsObj->color) << "|" << to_str(dsObj->license) + << "|" << to_str(dsObj->region) << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_PERSON: { + NvDsPersonObject *dsObj = (NvDsPersonObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->apparel) + << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_FACE: { + NvDsFaceObject *dsObj = (NvDsFaceObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->glasses) + << "|" << to_str(dsObj->facialhair) << "|" << to_str(dsObj->name) << "|" + << "|" << to_str(dsObj->eyecolor) << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_VEHICLE_EXT: { + NvDsVehicleObjectExt *dsObj = (NvDsVehicleObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->type) << "|" << to_str(dsObj->make) << "|" + << to_str(dsObj->model) << "|" << to_str(dsObj->color) << "|" << to_str(dsObj->license) + << "|" << to_str(dsObj->region) << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + case NVDS_OBJECT_TYPE_PERSON_EXT: { + NvDsPersonObjectExt *dsObj = (NvDsPersonObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->apparel) + << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + case NVDS_OBJECT_TYPE_FACE_EXT: { + NvDsFaceObjectExt *dsObj = (NvDsFaceObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->glasses) + << "|" << to_str(dsObj->facialhair) << "|" << to_str(dsObj->name) << "|" + << "|" << to_str(dsObj->eyecolor) << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + default: + cout << "Object type (" << meta->objType << ") not implemented" << endl; + break; + } + } + + if (objectMask) { + if (maskArray == NULL) + maskArray = json_array_new (); + generate_mask_array (meta, maskArray, objectMask); + } + + json_array_add_string_element (jArray, ss.str().c_str()); + } + + // It is assumed that all events / objects are associated with same frame. + // Therefore ts / sensorId / frameId of first object can be used. + + jobject = json_object_new (); + json_object_set_string_member (jobject, "version", "4.0"); + json_object_set_string_member (jobject, "id", to_string(events[0].metadata->frameId).c_str()); + json_object_set_string_member (jobject, "@timestamp", events[0].metadata->ts); + if (events[0].metadata->sensorStr) { + json_object_set_string_member (jobject, "sensorId", events[0].metadata->sensorStr); + } else if ((NvDsPayloadPriv *) privData) { + json_object_set_string_member (jobject, "sensorId", + to_str((gchar *) sensor_id_to_str (privData, events[0].metadata->sensorId))); + } else { + json_object_set_string_member (jobject, "sensorId", "0"); + } + + json_object_set_array_member (jobject, "objects", jArray); + if (maskArray && json_array_get_length (maskArray) > 0) + json_object_set_array_member (jobject, "masks", maskArray); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, jobject); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (jobject); + + return message; +} diff --git a/legacy_apps/deepstream-retail-analytics/nvmsgconv/nvmsgconv.cpp b/legacy_apps/deepstream-retail-analytics/nvmsgconv/nvmsgconv.cpp new file mode 100755 index 0000000..8bd10d2 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/nvmsgconv/nvmsgconv.cpp @@ -0,0 +1,268 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "nvmsgconv.h" +#include "deepstream_schema.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + + +NvDsMsg2pCtx* nvds_msg2p_ctx_create (const gchar *file, NvDsPayloadType type) +{ + NvDsMsg2pCtx *ctx = NULL; + string str; + bool retVal = true; + + /* + * Need to parse configuration / CSV files to get static properties of + * components (e.g. sensor, place etc.) in case of full deepstream schema. + */ + if (type == NVDS_PAYLOAD_DEEPSTREAM) { + g_return_val_if_fail (file, NULL); + + ctx = new NvDsMsg2pCtx; + ctx->privData = create_deepstream_schema_ctx(); + + if (g_str_has_suffix (file, ".csv")) { + retVal = nvds_msg2p_parse_csv (ctx->privData, file); + } else if (g_str_has_suffix (file, ".yml") || + g_str_has_suffix (file, ".yaml")) { + retVal = nvds_msg2p_parse_yaml (ctx->privData, file); + } else { + retVal = nvds_msg2p_parse_key_value (ctx->privData, file); + } + } else { + ctx = new NvDsMsg2pCtx; + /* If configuration file is provided for minimal schema, + * parse it for static values. + */ + if (file) { + ctx->privData = create_deepstream_schema_ctx(); + if (g_str_has_suffix (file, ".yml") || + g_str_has_suffix (file, ".yaml")) { + retVal = nvds_msg2p_parse_yaml (ctx->privData, file); + } else { + retVal = nvds_msg2p_parse_key_value (ctx->privData, file); + } + } else { + ctx->privData = nullptr; + retVal = true; + } + } + + ctx->payloadType = type; + + if (!retVal) { + cout << "Error in creating instance" << endl; + + if (ctx && ctx->privData) + destroy_deepstream_schema_ctx(ctx->privData); + + if (ctx) { + delete ctx; + ctx = NULL; + } + } + return ctx; +} + +void nvds_msg2p_ctx_destroy (NvDsMsg2pCtx *ctx) +{ + destroy_deepstream_schema_ctx(ctx->privData); + ctx->privData = nullptr; + delete ctx; +} + +NvDsPayload** +nvds_msg2p_generate_multiple (NvDsMsg2pCtx *ctx, NvDsEvent *events, guint eventSize, + guint *payloadCount) +{ + gchar *message = NULL; + gint len = 0; + NvDsPayload **payloads = NULL; + *payloadCount = 0; + //Set how many payloads are being sent back to the plugin + payloads = (NvDsPayload **) g_malloc0 (sizeof (NvDsPayload*) * 1); + + if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM) { + message = generate_event_message (ctx->privData, events->metadata); + if (message) { + payloads[*payloadCount]= (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + len = strlen (message); + // Remove '\0' character at the end of string and just copy the content. + payloads[*payloadCount]->payload = g_memdup (message, len); + payloads[*payloadCount]->payloadSize = len; + ++(*payloadCount); + g_free (message); + } + } else if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM_MINIMAL) { + message = generate_event_message_minimal (ctx->privData, events, eventSize); + if (message) { + len = strlen (message); + payloads[*payloadCount] = (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + // Remove '\0' character at the end of string and just copy the content. + payloads[*payloadCount]->payload = g_memdup (message, len); + payloads[*payloadCount]->payloadSize = len; + ++(*payloadCount); + g_free (message); + } + } else if (ctx->payloadType == NVDS_PAYLOAD_CUSTOM) { + payloads[*payloadCount] = (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + payloads[*payloadCount]->payload = (gpointer) g_strdup ("CUSTOM Schema"); + payloads[*payloadCount]->payloadSize = strlen ((char *)payloads[*payloadCount]->payload) + 1; + ++(*payloadCount); + } else + payloads = NULL; + + return payloads; +} + +NvDsPayload* +nvds_msg2p_generate (NvDsMsg2pCtx *ctx, NvDsEvent *events, guint size) +{ + gchar *message = NULL; + gint len = 0; + NvDsPayload *payload = (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + + if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM) { + message = generate_event_message (ctx->privData, events->metadata); + if (message) { + len = strlen (message); + // Remove '\0' character at the end of string and just copy the content. + payload->payload = g_memdup (message, len); + payload->payloadSize = len; + g_free (message); + } + } else if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM_MINIMAL) { + message = generate_event_message_minimal (ctx->privData, events, size); + if (message) { + len = strlen (message); + // Remove '\0' character at the end of string and just copy the content. + payload->payload = g_memdup (message, len); + payload->payloadSize = len; + g_free (message); + } + } else if (ctx->payloadType == NVDS_PAYLOAD_CUSTOM) { + payload->payload = (gpointer) g_strdup ("CUSTOM Schema"); + payload->payloadSize = strlen ((char *)payload->payload) + 1; + } else + payload->payload = NULL; + + return payload; +} + +NvDsPayload* +nvds_msg2p_generate_new (NvDsMsg2pCtx *ctx, void *metadataInfo) +{ + gchar *message = NULL; + gint len = 0; + NvDsMsg2pMetaInfo *meta_info = (NvDsMsg2pMetaInfo *) metadataInfo; + NvDsFrameMeta *frame_meta = (NvDsFrameMeta*) meta_info->frameMeta; + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *) meta_info->objMeta; + + NvDsPayload *payload = (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + + if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM) { + message = generate_dsmeta_message(ctx->privData, frame_meta, obj_meta); + if (message) { + len = strlen (message); + // Remove '\0' character at the end of string and just copy the content. + payload->payload = g_memdup (message, len); + payload->payloadSize = len; + g_free (message); + } + } + else if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM_MINIMAL) { + message = generate_dsmeta_message_minimal (ctx->privData, frame_meta); + if (message) { + len = strlen (message); + // Remove '\0' character at the end of string and just copy the content. + payload->payload = g_memdup (message, len); + payload->payloadSize = len; + g_free (message); + } + } else if (ctx->payloadType == NVDS_PAYLOAD_CUSTOM) { + payload->payload = (gpointer) g_strdup ("CUSTOM Schema"); + payload->payloadSize = strlen ((char *)payload->payload) + 1; + } else + payload->payload = NULL; + + return payload; +} + +NvDsPayload** +nvds_msg2p_generate_multiple_new (NvDsMsg2pCtx *ctx, void *metadataInfo, guint *payloadCount) +{ + gchar *message = NULL; + gint len = 0; + NvDsPayload **payloads = NULL; + *payloadCount = 0; + //Set how many payloads are being sent back to the plugin + payloads = (NvDsPayload **) g_malloc0 (sizeof (NvDsPayload*) * 1); + + NvDsMsg2pMetaInfo *meta_info = (NvDsMsg2pMetaInfo *) metadataInfo; + NvDsFrameMeta *frame_meta = (NvDsFrameMeta*) meta_info->frameMeta; + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *) meta_info->objMeta; + + if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM) { + message = generate_dsmeta_message(ctx->privData, frame_meta, obj_meta); + if (message) { + payloads[*payloadCount]= (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + len = strlen (message); + // Remove '\0' character at the end of string and just copy the content. + payloads[*payloadCount]->payload = g_memdup (message, len); + payloads[*payloadCount]->payloadSize = len; + ++(*payloadCount); + g_free (message); + } + } else if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM_MINIMAL) { + message = generate_dsmeta_message_minimal (ctx->privData, frame_meta); + if (message) { + len = strlen (message); + payloads[*payloadCount] = (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + // Remove '\0' character at the end of string and just copy the content. + payloads[*payloadCount]->payload = g_memdup (message, len); + payloads[*payloadCount]->payloadSize = len; + ++(*payloadCount); + g_free (message); + } + } else if (ctx->payloadType == NVDS_PAYLOAD_CUSTOM) { + payloads[*payloadCount] = (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + payloads[*payloadCount]->payload = (gpointer) g_strdup ("CUSTOM Schema"); + payloads[*payloadCount]->payloadSize = strlen ((char *)payloads[*payloadCount]->payload) + 1; + ++(*payloadCount); + } else + payloads = NULL; + + return payloads; +} + +void +nvds_msg2p_release (NvDsMsg2pCtx *ctx, NvDsPayload *payload) +{ + g_free (payload->payload); + g_free (payload); +} diff --git a/legacy_apps/deepstream-retail-analytics/nvmsgconv/nvmsgconv.h b/legacy_apps/deepstream-retail-analytics/nvmsgconv/nvmsgconv.h new file mode 100755 index 0000000..246ba89 --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/nvmsgconv/nvmsgconv.h @@ -0,0 +1,170 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** + * @file + * NVIDIA DeepStream: Message Schema Generation Library Interface + * + * @b Description: This file specifies the NVIDIA DeepStream message schema generation + * library interface. + */ + +#ifndef NVMSGCONV_H_ +#define NVMSGCONV_H_ + +#include "nvdsmeta_schema.h" +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + + +/** + * @ref NvDsMsg2pCtx is structure for library context. + */ +typedef struct NvDsMsg2pCtx { + /** type of payload to be generated. */ + NvDsPayloadType payloadType; + + /** private to component. Don't change this field. */ + gpointer privData; +} NvDsMsg2pCtx; + +/** + * @ref NvDsMsg2pMetaInfo is structure to hold + * the NvDs metadata related information + to be processed to generate payloads + */ + +typedef struct { + /** Holds the object metadata */ + void *objMeta; + /** Holds the frame metadata */ + void *frameMeta; + /** media type: (ex: audio, video) */ + gchar *mediaType; +} NvDsMsg2pMetaInfo; + +/** + * This function initializes the library with user defined options mentioned + * in the file and returns the handle to the context. + * Static fields which should be part of message payload can be added to + * file instead of frame metadata. + * + * @param[in] file name of file to read static properties from. + * @param[in] type type of payload to be generated. + * + * @return pointer to library context created. This context should be used in + * other functions of library and should be freed with + * @ref nvds_msg2p_ctx_destroy + */ +NvDsMsg2pCtx* nvds_msg2p_ctx_create (const gchar *file, NvDsPayloadType type); + +/** + * Release the resources allocated during context creation. + * + * @param[in] ctx pointer to library context. + */ +void nvds_msg2p_ctx_destroy (NvDsMsg2pCtx *ctx); + +/** + * This function will parse the @ref NvDsEventMsgMeta and will generate message + * payload. Payload will be combination of static values read from + * configuration file and dynamic values received in meta. + * Payload will be generated based on the @ref NvDsPayloadType type provided + * in context creation (e.g. Deepstream, Custom etc.). + * + * @param[in] ctx pointer to library context. + * @param[in] events pointer to array of event objects. + * @param[in] size number of objects in array. + * + * @return pointer to @ref NvDsPayload generated or NULL in case of error. + * This payload should be freed with @ref nvds_msg2p_release + */ +NvDsPayload* +nvds_msg2p_generate (NvDsMsg2pCtx *ctx, NvDsEvent *events, guint size); + +/** + * This function will parse the @ref NvDsEventMsgMeta and will generate multiple + * message payloads. Payloads will be combination of static values read from + * configuration file and dynamic values received in meta. + * Payloads will be generated based on the @ref NvDsPayloadType type provided + * in context creation (e.g. Deepstream, Custom etc.). + * + * @param[in] ctx pointer to library context. + * @param[in] events pointer to array of event objects. + * @param[in] size number of objects in array. + * @param[out] payloadCount number of payloads being returned by the function. + * + * @return pointer to @ref array of NvDsPayload pointers generated or NULL in + * case of error. The number of payloads in the array is returned through + * payloadCount. This pointer should be freed by calling g_free() and the + * individual payloads should be freed with @ref nvds_msg2p_release + */ +NvDsPayload** +nvds_msg2p_generate_multiple (NvDsMsg2pCtx *ctx, NvDsEvent *events, guint size, guint *payloadCount); + +/** + * This function will parse the @ref NvDsMsg2pMetaInfo and will generate + * message payloads. Payloads will be combination of static values read from + * configuration file and the deepstream metadata fields passed @ref NvDsMsg2pMetaInfo + * Payloads will be generated based on the @ref NvDsPayloadType type provided + * in context creation (e.g. Deepstream, Custom etc.). + * + * @param[in] ctx pointer to library context. + * @param[in] pointer to type NvDsMsg2pMetaInfo + * + * @return pointer to @ref NvDsPayload generated or NULL in case of error. + * This payload should be freed with @ref nvds_msg2p_release + */ +NvDsPayload* +nvds_msg2p_generate_new (NvDsMsg2pCtx *ctx, void *metadataInfo); + +/** + * This function will parse the @ref NvDsMsg2pMetaInfo and will generate multiple + * message payloads. Payloads will be combination of static values read from + * configuration file and the deepstream metadata fields passed @ref NvDsMsg2pMetaInfo + * Payloads will be generated based on the @ref NvDsPayloadType type provided + * in context creation (e.g. Deepstream, Custom etc.). + * + * @param[in] ctx pointer to library context. + * @param[in] pointer to type NvDsMsg2pMetaInfo + * @param[out] payloadCount number of payloads being returned by the function. + * + * @return pointer to @ref array of NvDsPayload pointers generated or NULL in + * case of error. The number of payloads in the array is returned through + * payloadCount. This pointer should be freed by calling g_free() and the + * individual payloads should be freed with @ref nvds_msg2p_release + */ +NvDsPayload** +nvds_msg2p_generate_multiple_new (NvDsMsg2pCtx *ctx, void *metadataInfo, guint *payloadCount); + +/** + * This function should be called to release memory allocated for payload. + * + * @param[in] ctx pointer to library context. + * @param[in] payload pointer to object that needs to be released. + */ +void nvds_msg2p_release (NvDsMsg2pCtx *ctx, NvDsPayload *payload); + +#ifdef __cplusplus +} +#endif +#endif /* NVMSGCONV_H_ */ diff --git a/legacy_apps/deepstream-retail-analytics/retail_iva.c b/legacy_apps/deepstream-retail-analytics/retail_iva.c new file mode 100755 index 0000000..cc7978a --- /dev/null +++ b/legacy_apps/deepstream-retail-analytics/retail_iva.c @@ -0,0 +1,899 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gstnvdsmeta.h" +#include "nvdsmeta_schema.h" +#include "nvds_yml_parser.h" + +#define MAX_DISPLAY_LEN 64 +#define MAX_TIME_STAMP_LEN 32 + +// Config files for detectors, tracker and message broker +#define PGIE_CONFIG_FILE "configs/pgie_config_peoplenet.txt" +#define SGIE_CONFIG_FILE "configs/basket_classifier.txt" +#define TRACKER_CONFIG_FILE "configs/dstest4_tracker_config.txt" +#define MSCONV_CONFIG_FILE "configs/dstest4_msgconv_config.txt" + +// Primary detector class IDs +#define PGIE_CLASS_ID_PERSON 0 +#define PGIE_CLASS_ID_FACE 1 +#define PGIE_CLASS_ID_BAG 2 + +// Properties of nvstreammux +#define MUXER_OUTPUT_WIDTH 1920 +#define MUXER_OUTPUT_HEIGHT 1080 + +#define MUXER_BATCH_TIMEOUT_USEC 40000 + +#define CHECK_ERROR(error) \ + if (error) { \ + g_printerr("Error while parsing config file: %s\n", error->message); \ + goto done; \ + } + +// Keys to read tracker config file +#define CONFIG_GROUP_TRACKER "tracker" +#define CONFIG_GROUP_TRACKER_WIDTH "tracker-width" +#define CONFIG_GROUP_TRACKER_HEIGHT "tracker-height" +#define CONFIG_GROUP_TRACKER_LL_CONFIG_FILE "ll-config-file" +#define CONFIG_GROUP_TRACKER_LL_LIB_FILE "ll-lib-file" +#define CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS "enable-batch-process" +#define CONFIG_GPU_ID "gpu-id" + +// Define variables to store config parameters +static gchar *cfg_file = NULL; +static gchar *input_file = NULL; +static gchar *topic = NULL; +static gchar *conn_str = NULL; +static gchar *proto_lib = NULL; +static gint schema_type = 0; +static gint msg2p_meta = 0; +static gint frame_interval = 15; +static gboolean display_off = FALSE; + +// Array to store classes detected by PGIE +gchar pgie_classes_str[3][32] = {"Person", "Bag", "Face"}; + +GOptionEntry entries[] = { + {"cfg-file", 'c', 0, G_OPTION_ARG_FILENAME, &cfg_file, + "Set the adaptor config file. Optional if connection string has relevant details.", + NULL}, + {"input-file", 'i', 0, G_OPTION_ARG_FILENAME, &input_file, + "Set the input H264 file", NULL}, + {"topic", 't', 0, G_OPTION_ARG_STRING, &topic, + "Name of message topic. Optional if it is part of connection string or config file.", + NULL}, + {"conn-str", 0, 0, G_OPTION_ARG_STRING, &conn_str, + "Connection string of backend server. Optional if it is part of config file.", + NULL}, + {"proto-lib", 'p', 0, G_OPTION_ARG_STRING, &proto_lib, + "Absolute path of adaptor library", NULL}, + {"schema", 's', 0, G_OPTION_ARG_INT, &schema_type, + "Type of message schema (0=Full, 1=minimal), default=0", NULL}, + {"msg2p-meta", 0, 0, G_OPTION_ARG_INT, &msg2p_meta, + "msg2payload generation metadata type (0=Event Msg meta, 1=nvds meta), default=0", + NULL}, + {"frame-interval", 0, 0, G_OPTION_ARG_INT, &frame_interval, + "Frame interval at which payload is generated , default=30", NULL}, + {"no-display", 0, 0, G_OPTION_ARG_NONE, &display_off, "Disable display", + NULL}, + {NULL} +}; + +// Function to check if an input file is a YAML file +#define IS_YAML(file) (g_str_has_suffix (file, ".yml") || g_str_has_suffix (file, ".yaml")) + +gint frame_number = 0; + +// Function to generate timestamp for kafka message +static void +generate_ts_rfc3339 (char *buf, int buf_size) +{ + time_t tloc; + struct tm tm_log; + struct timespec ts; + char strmsec[6]; //.nnnZ\0 + + clock_gettime (CLOCK_REALTIME, &ts); + memcpy (&tloc, (void *) (&ts.tv_sec), sizeof (time_t)); + gmtime_r (&tloc, &tm_log); + strftime (buf, buf_size, "%Y-%m-%dT%H:%M:%S", &tm_log); + int ms = ts.tv_nsec / 1000000; + g_snprintf (strmsec, sizeof (strmsec), ".%.3dZ", ms); + strncat (buf, strmsec, buf_size); +} + +// Callback function for deep-copying NvDsEventMsgMeta struct +static gpointer +meta_copy_func (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + NvDsEventMsgMeta *dstMeta = NULL; + + dstMeta = g_memdup (srcMeta, sizeof (NvDsEventMsgMeta)); + + if (srcMeta->ts) + dstMeta->ts = g_strdup (srcMeta->ts); + + if (srcMeta->sensorStr) + dstMeta->sensorStr = g_strdup (srcMeta->sensorStr); + + if (srcMeta->objSignature.size > 0) { + dstMeta->objSignature.signature = g_memdup (srcMeta->objSignature.signature, + srcMeta->objSignature.size); + dstMeta->objSignature.size = srcMeta->objSignature.size; + } + + if (srcMeta->objectId) { + dstMeta->objectId = g_strdup (srcMeta->objectId); + } + + if (srcMeta->extMsgSize > 0) { + if (srcMeta->objType == NVDS_OBJECT_TYPE_PERSON) { + NvDsPersonObject *srcObj = (NvDsPersonObject *) srcMeta->extMsg; + NvDsPersonObject *obj = + (NvDsPersonObject *) g_malloc0 (sizeof (NvDsPersonObject)); + + if (srcObj->hasBasket) + obj->hasBasket = g_strdup (srcObj->hasBasket); + + dstMeta->extMsg = obj; + dstMeta->extMsgSize = sizeof (NvDsPersonObject); + } + } + + return dstMeta; +} + +// Callback function to free NvDsEventMsgMeta struct +static void +meta_free_func (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + + g_free (srcMeta->ts); + g_free (srcMeta->sensorStr); + + if (srcMeta->objSignature.size > 0) { + g_free (srcMeta->objSignature.signature); + srcMeta->objSignature.size = 0; + } + + if (srcMeta->objectId) { + g_free (srcMeta->objectId); + } + + if (srcMeta->extMsgSize > 0) { + if (srcMeta->objType == NVDS_OBJECT_TYPE_PERSON) { + NvDsPersonObject *obj = (NvDsPersonObject *) srcMeta->extMsg; + + if (obj->hasBasket) + g_free (obj->hasBasket); + } + g_free (srcMeta->extMsg); + srcMeta->extMsgSize = 0; + } + g_free (user_meta->user_meta_data); + user_meta->user_meta_data = NULL; +} + +// Function to return label from classifier metadata +static gchar * +get_first_result_label (NvDsClassifierMeta * classifierMeta) +{ + GList *n; + // Iterate through all the secondary labels stored in classifierMeta + // Refer to deepstream-test5 + for (n = classifierMeta->label_info_list; n != NULL; n = n->next) { + NvDsLabelInfo *labelInfo = (NvDsLabelInfo *) (n->data); + if (labelInfo->result_label[0] != '\0') { + return g_strdup (labelInfo->result_label); + } + } + return NULL; +} + +// Function to generate metadata for person object type +static void +generate_person_meta (gpointer data) +{ + NvDsPersonObject *obj = (NvDsPersonObject *) data; +} + +// Function to generate event message metadata from object metadata +static void +generate_event_msg_meta (gpointer data, gint class_id, + NvDsObjectMeta * obj_params) +{ + NvDsEventMsgMeta *meta = (NvDsEventMsgMeta *) data; + meta->sensorId = 0; + meta->placeId = 0; + meta->moduleId = 0; + meta->sensorStr = g_strdup ("sensor-0"); + + meta->ts = (gchar *) g_malloc0 (MAX_TIME_STAMP_LEN + 1); + meta->objectId = (gchar *) g_malloc0 (MAX_LABEL_SIZE); + + strncpy (meta->objectId, obj_params->obj_label, MAX_LABEL_SIZE); + + generate_ts_rfc3339 (meta->ts, MAX_TIME_STAMP_LEN); + + // * This demonstrates how to attach custom objects. + // * Any custom object as per requirement can be generated and attached + // * like NvDsVehicleObject / NvDsPersonObject. Then that object should + // * be handled in payload generator library (nvmsgconv.cpp) accordingly. + + // Attach the secondary label to the person object detected + + if (class_id == PGIE_CLASS_ID_PERSON) { + meta->type = NVDS_EVENT_ENTRY; + meta->objType = NVDS_OBJECT_TYPE_PERSON; + meta->objClassId = PGIE_CLASS_ID_PERSON; + + NvDsPersonObject *obj = + (NvDsPersonObject *) g_malloc0 (sizeof (NvDsPersonObject)); + generate_person_meta (obj); + + GList *l; + for (l = obj_params->classifier_meta_list; l!= NULL; l = l->next) { + NvDsClassifierMeta *classifierMeta = (NvDsClassifierMeta *) (l->data); + obj->hasBasket = get_first_result_label(classifierMeta); + } + + meta->extMsg = obj; + meta->extMsgSize = sizeof (NvDsPersonObject); + } +} + +// Probe function to generate OSD data +static GstPadProbeReturn +osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, + gpointer u_data) +{ + // Uncomment lines containing is_first_object send kafka messages + // for only the first detection in the frame. + + GstBuffer *buf = (GstBuffer *) info->data; + NvDsFrameMeta *frame_meta = NULL; + NvOSD_TextParams *txt_params = NULL; + guint person_count = 0; + // gboolean is_first_object = TRUE; + NvDsMetaList *l_frame, *l_obj; + gchar *sgie_label=NULL; + + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf); + if (!batch_meta) { + // No batch meta attached. + return GST_PAD_PROBE_OK; + } + + for (l_frame = batch_meta->frame_meta_list; l_frame; l_frame = l_frame->next) { + frame_meta = (NvDsFrameMeta *) l_frame->data; + + if (frame_meta == NULL) { + // Ignore Null frame meta. + continue; + } + + // is_first_object = TRUE; + + for (l_obj = frame_meta->obj_meta_list; l_obj; l_obj = l_obj->next) { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *) l_obj->data; + + if (obj_meta == NULL) { + // Ignore Null object. + continue; + } + + GList *l; + sgie_label = "NULL"; + for (l = obj_meta->classifier_meta_list; l!= NULL; l = l->next) { + NvDsClassifierMeta *classifierMeta = (NvDsClassifierMeta *) (l->data); + sgie_label = get_first_result_label(classifierMeta); + } + + txt_params = &(obj_meta->text_params); + if (txt_params->display_text) + g_free (txt_params->display_text); + + txt_params->display_text = g_malloc0 (MAX_DISPLAY_LEN); + + g_snprintf (txt_params->display_text, MAX_DISPLAY_LEN, "%s %ld %s", + pgie_classes_str[obj_meta->class_id], obj_meta->object_id, sgie_label); /* Person 12 hasBasket */ + + person_count++; + + /* Now set the offsets where the string should appear */ + txt_params->x_offset = obj_meta->rect_params.left; + txt_params->y_offset = obj_meta->rect_params.top - 25; + + /* Font , font-color and font-size */ + txt_params->font_params.font_name = "Serif"; + txt_params->font_params.font_size = 10; + txt_params->font_params.font_color.red = 1.0; + txt_params->font_params.font_color.green = 1.0; + txt_params->font_params.font_color.blue = 1.0; + txt_params->font_params.font_color.alpha = 1.0; + + /* Text background color */ + txt_params->set_bg_clr = 1; + txt_params->text_bg_clr.red = 0.0; + txt_params->text_bg_clr.green = 0.0; + txt_params->text_bg_clr.blue = 0.0; + txt_params->text_bg_clr.alpha = 1.0; + + // * Ideally NVDS_EVENT_MSG_META should be attached to buffer by the + // * component implementing detection / recognition logic. + // * Here it demonstrates how to use / attach that meta data. + + if (/*is_first_object && */ !(frame_number % frame_interval)) { + /* Frequency of messages to be send will be based on use case. + * Here message is being sent for first object every frame_interval(default=15). + */ + if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) { + NvDsEventMsgMeta *msg_meta = + (NvDsEventMsgMeta *) g_malloc0 (sizeof (NvDsEventMsgMeta)); + msg_meta->bbox.top = obj_meta->rect_params.top; + msg_meta->bbox.left = obj_meta->rect_params.left; + msg_meta->bbox.width = obj_meta->rect_params.width; + msg_meta->bbox.height = obj_meta->rect_params.height; + msg_meta->frameId = frame_number; + msg_meta->trackingId = obj_meta->object_id; + msg_meta->confidence = obj_meta->confidence; + generate_event_msg_meta (msg_meta, obj_meta->class_id, obj_meta); + + NvDsUserMeta *user_event_meta = + nvds_acquire_user_meta_from_pool (batch_meta); + if (user_event_meta) { + user_event_meta->user_meta_data = (void *) msg_meta; + user_event_meta->base_meta.meta_type = NVDS_EVENT_MSG_META; + user_event_meta->base_meta.copy_func = + (NvDsMetaCopyFunc) meta_copy_func; + user_event_meta->base_meta.release_func = + (NvDsMetaReleaseFunc) meta_free_func; + nvds_add_user_meta_to_frame (frame_meta, user_event_meta); + } else { + g_print ("Error in attaching event meta to buffer\n"); + } + } + // is_first_object = FALSE; + } + } + } + g_print ("Frame Number = %d " + "Person Count = %d\n", + frame_number, person_count); + frame_number++; + + return GST_PAD_PROBE_OK; +} + +static gboolean +bus_call (GstBus * bus, GstMessage * msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + g_print ("End of stream\n"); + g_main_loop_quit (loop); + break; + case GST_MESSAGE_ERROR:{ + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + default: + break; + } + return TRUE; +} + +static gchar * +get_absolute_file_path (gchar *cfg_file_path, gchar *file_path) +{ + gchar abs_cfg_path[PATH_MAX + 1]; + gchar *abs_file_path; + gchar *delim; + + if (file_path && file_path[0] == '/') { + return file_path; + } + + if (!realpath (cfg_file_path, abs_cfg_path)) { + g_free (file_path); + return NULL; + } + + // Return absolute path of config file if file_path is NULL. + if (!file_path) { + abs_file_path = g_strdup (abs_cfg_path); + return abs_file_path; + } + + delim = g_strrstr (abs_cfg_path, "/"); + *(delim + 1) = '\0'; + + abs_file_path = g_strconcat (abs_cfg_path, file_path, NULL); + g_free (file_path); + + return abs_file_path; +} + +// Function to read tracker config file and set properties of the GstElement +static gboolean +set_tracker_properties (GstElement *nvtracker) +{ + gboolean ret = FALSE; + GError *error = NULL; + gchar **keys = NULL; + gchar **key = NULL; + GKeyFile *key_file = g_key_file_new (); + + if (!g_key_file_load_from_file (key_file, TRACKER_CONFIG_FILE, G_KEY_FILE_NONE, + &error)) { + g_printerr ("Failed to load config file: %s\n", error->message); + return FALSE; + } + + keys = g_key_file_get_keys (key_file, CONFIG_GROUP_TRACKER, NULL, &error); + CHECK_ERROR (error); + + for (key = keys; *key; key++) { + if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_WIDTH)) { + gint width = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_WIDTH, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "tracker-width", width, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_HEIGHT)) { + gint height = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_HEIGHT, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "tracker-height", height, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GPU_ID)) { + guint gpu_id = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GPU_ID, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "gpu_id", gpu_id, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_LL_CONFIG_FILE)) { + char* ll_config_file = get_absolute_file_path (TRACKER_CONFIG_FILE, + g_key_file_get_string (key_file, + CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_LL_CONFIG_FILE, &error)); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "ll-config-file", ll_config_file, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_LL_LIB_FILE)) { + char* ll_lib_file = get_absolute_file_path (TRACKER_CONFIG_FILE, + g_key_file_get_string (key_file, + CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_LL_LIB_FILE, &error)); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "ll-lib-file", ll_lib_file, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS)) { + gboolean enable_batch_process = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "enable_batch_process", + enable_batch_process, NULL); + } else { + g_printerr ("Unknown key '%s' for group [%s]", *key, + CONFIG_GROUP_TRACKER); + } + } + + ret = TRUE; + done: + if (error) { + g_error_free (error); + } + if (keys) { + g_strfreev (keys); + } + if (!ret) { + g_printerr ("%s failed", __func__); + } + return ret; +} + +static void +check_gst_element_creation_success(GstElement *element, gchar *component_name) +{ + if (!element) { + g_printerr("%s element could not be created\n", component_name); + } +} + +int +main (int argc, char *argv[]) +{ + // Initialize elements of the DS pipeline + GMainLoop *loop = NULL; + GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, + *decoder = NULL, *sink = NULL, *pgie = NULL, *sgie = NULL, + *nvvidconv = NULL, *nvosd = NULL, *nvstreammux = NULL, *nvtracker = NULL; + GstElement *msgconv = NULL, *msgbroker = NULL, *tee = NULL; + GstElement *queue1 = NULL, *queue2 = NULL; + GstElement *transform = NULL; + GstBus *bus = NULL; + guint bus_watch_id; + GstPad *osd_sink_pad = NULL; + GstPad *tee_render_pad = NULL; + GstPad *tee_msg_pad = NULL; + GstPad *sink_pad = NULL; + GstPad *src_pad = NULL; + GOptionContext *ctx = NULL; + GOptionGroup *group = NULL; + GError *error = NULL; + + int current_device = -1; + cudaGetDevice (¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + ctx = g_option_context_new("DeepStream Retail IVA"); + group = g_option_group_new("RetailIVA", NULL, NULL, NULL, NULL); + g_option_group_add_entries(group, entries); + + g_option_context_set_main_group(ctx, group); + g_option_context_add_group(ctx, gst_init_get_option_group()); + + if (!g_option_context_parse(ctx, &argc, &argv, &error)) { + g_option_context_free (ctx); + g_printerr("%s", error->message); + return -1; + } + + if (!proto_lib || !input_file) { + if (argc > 1 && !IS_YAML (argv[1])) { + g_printerr ("missing arguments\n"); + g_printerr ("Usage: %s \n", argv[0]); + g_printerr + ("Usage: %s -i -p --conn-str=\n", + argv[0]); + return -1; + } else if (!argv[1]) { + g_printerr ("missing arguments\n"); + g_printerr ("Usage: %s \n", argv[0]); + g_printerr + ("Usage: %s -i -p --conn-str=\n", + argv[0]); + return -1; + } + } + + loop = g_main_loop_new (NULL, FALSE); + + // Create gstreamer elements + // Create pipeline element that will hold connection of all other elements + pipeline = gst_pipeline_new ("retail-iva-pipeline"); + + // Source element for reading the input file + source = gst_element_factory_make ("filesrc", "file-source"); + + // h264parser to parse input file + h264parser = gst_element_factory_make("h264parse", "h264-parser"); + + // nvdec_h264 for hardware accelerated decoding on GPU + decoder = gst_element_factory_make("nvv4l2decoder", "nvv4l2-decoder"); + + nvstreammux = gst_element_factory_make ("nvstreammux", "nvstreammux"); + + // nvinfer to run inferencing on decoder's output - PGIE + pgie = gst_element_factory_make("nvinfer", "primary-inference-engine"); + + // nvinfer to run inferencing on PGIE's output - SGIE + sgie = gst_element_factory_make("nvinfer", "secondary-inference-engine"); + + // tracker to track objects detected by PGIE + nvtracker = gst_element_factory_make("nvtracker", "tracker"); + + // converter to convert from NV12 to RGBA + nvvidconv = gst_element_factory_make("nvvideoconvert", "nvvideo-converter"); + + // osd to draw on converted RGBA buffer + nvosd = gst_element_factory_make("nvdsosd", "nv-onscreendisplay"); + + // Create message converter to generate payload from buffer metadata + msgconv = gst_element_factory_make("nvmsgconv", "nvmsg-converter"); + + // Create message broker to send payload to kafka server + msgbroker = gst_element_factory_make("nvmsgbroker", "nvmsg-broker"); + + // Create teeo to render buffer and send messages simultaneously + tee = gst_element_factory_make("tee", "nvsink-tee"); + + // Create queues + queue1 = gst_element_factory_make("queue", "nvtee-que1"); + queue2 = gst_element_factory_make("queue", "nvtee-que2"); + + // Finally render the osd output + if (display_off) { + sink = gst_element_factory_make("fakesink", "nvvideo-renderer"); + + } else { + sink = gst_element_factory_make("nveglglessink", "nvvideo-renderer"); + if (prop.integrated) { + transform = + gst_element_factory_make("nvegltransform", "nvegl-transform"); + if (!transform) { + g_printerr("nvegltransform element could not be created. Exiting\n"); + return -1; + } + } + } + + + // Check if the pipeline and all elements are created + if (!pipeline || !source || !h264parser || !decoder || !nvstreammux || !pgie || + !sgie || !nvtracker || !nvvidconv || !nvosd || !msgconv || !msgbroker || + !tee || !queue1 || !queue2 || !sink) { + // Check which element was not created + check_gst_element_creation_success(pipeline, "pipeline"); + check_gst_element_creation_success(source, "source"); + check_gst_element_creation_success(h264parser, "h264parser"); + check_gst_element_creation_success(decoder, "decoder"); + check_gst_element_creation_success(nvstreammux, "nvstreammux"); + check_gst_element_creation_success(pgie, "pgie"); + check_gst_element_creation_success(sgie, "sgie"); + check_gst_element_creation_success(nvtracker, "nvtracker"); + check_gst_element_creation_success(nvvidconv, "nvvidconv"); + check_gst_element_creation_success(nvosd, "nvosd"); + check_gst_element_creation_success(msgconv, "msgconv"); + check_gst_element_creation_success(msgbroker, "msgbroker"); + check_gst_element_creation_success(tee, "tee"); + check_gst_element_creation_success(queue1, "queue1"); + check_gst_element_creation_success(queue2, "queue2"); + check_gst_element_creation_success(sink, "sink"); + g_printerr("One above element could not be created. Exiting \n"); + return -1; + } + + if (argc > 1 && IS_YAML (argv[1])) { + nvds_parse_file_source (source, argv[1], "source"); + nvds_parse_streammux (nvstreammux, argv[1], "streammux"); + + g_object_set (G_OBJECT (pgie), + "config-file-path", "configs/pgie_config_peoplenet.yml", NULL); + + g_object_set (G_OBJECT(sgie), + "config-file-path", "configs/basket_classifier.yml", NULL); + + g_object_set (G_OBJECT (msgconv), "config", "configs/dstest4_msgconv_config.yml", + NULL); + nvds_parse_msgconv (msgconv, argv[1], "msgconv"); + + nvds_parse_msgbroker (msgbroker, argv[1], "msgbroker"); + + if (display_off) + nvds_parse_file_sink (sink, argv[1], "sink"); + else + nvds_parse_egl_sink (sink, argv[1], "sink"); + + } else { + /* we set the input filename to the source element */ + g_object_set (G_OBJECT (source), "location", input_file, NULL); + + g_object_set (G_OBJECT (nvstreammux), "batch-size", 1, NULL); + + g_object_set (G_OBJECT (nvstreammux), "width", MUXER_OUTPUT_WIDTH, "height", + MUXER_OUTPUT_HEIGHT, + "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); + + /* Set all the necessary properties of the nvinfer element, + * the necessary ones are : */ + g_object_set (G_OBJECT (pgie), "config-file-path", PGIE_CONFIG_FILE, NULL); + + g_object_set (G_OBJECT (msgconv), "config", MSCONV_CONFIG_FILE, NULL); + g_object_set (G_OBJECT (msgconv), "payload-type", schema_type, NULL); + g_object_set (G_OBJECT (msgconv), "msg2p-newapi", msg2p_meta, NULL); + g_object_set (G_OBJECT (msgconv), "frame-interval", frame_interval, NULL); + + g_object_set (G_OBJECT (msgbroker), "proto-lib", proto_lib, + "conn-str", conn_str, "sync", FALSE, NULL); + + if (topic) { + g_object_set (G_OBJECT (msgbroker), "topic", topic, NULL); + } + + if (cfg_file) { + g_object_set (G_OBJECT (msgbroker), "config", cfg_file, NULL); + } + gchar *filepath; + filepath = g_strconcat("output",".mp4",NULL); + g_object_set (G_OBJECT (sink), "sync", TRUE, NULL); + } + + if (!set_tracker_properties(nvtracker)) { + g_printerr ("Failed to set tracker properties. Exiting \n"); + return -1; + } + + // Adding a message handler + bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline)); + bus_watch_id = gst_bus_add_watch(bus, bus_call, loop); + gst_object_unref(bus); + + // Setup a pipeline + // Add all elements to the pipeline + gst_bin_add_many(GST_BIN(pipeline), + source, h264parser, decoder, nvstreammux, pgie, sgie, nvtracker, + nvvidconv, nvosd, tee, queue1, queue2, msgconv, msgbroker, sink, NULL); + + if (prop.integrated) { + if (!display_off) + gst_bin_add (GST_BIN(pipeline), transform); + } + + // Link elements together + /* + file-source -> h264-parser -> nvh264-decoder -> nvstreammux -> + nvinfer -> nvtracker -> nvvidconv -> nvosd -> tee -> video-renderer + | + | -> msgconv -> msgbroker + */ + sink_pad = gst_element_get_request_pad(nvstreammux, "sink_0"); + if (!sink_pad) { + g_printerr("Streammux request sink pad failed. Exiting \n"); + return -1; + } + + src_pad = gst_element_get_static_pad(decoder, "src"); + if (!src_pad) { + g_printerr("Decoder request src pad failed. Exiting \n"); + return -1; + } + + if (gst_pad_link(src_pad, sink_pad) != GST_PAD_LINK_OK) { + g_printerr("Failed to link decoder to stream muxer. Exiting \n"); + return -1; + } + + gst_object_unref(sink_pad); + gst_object_unref(src_pad); + + if (!gst_element_link_many(source, h264parser, decoder, NULL)) { + g_printerr("Elements could not be linked. Exiting \n"); + return -1; + } + + if (!gst_element_link_many(nvstreammux, pgie, nvtracker, sgie, nvvidconv, nvosd, tee, NULL)) { + g_printerr ("Elements could not be linked. Exiting \n"); + return -1; + } + + if (!gst_element_link_many(queue1, msgconv, msgbroker, NULL)) { + g_printerr("Elements could not be linked. Exiting \n"); + return -1; + } + + if (prop.integrated) { + if (!display_off) { + if(!gst_element_link_many(queue2, transform, sink, NULL)) { + g_printerr("Elements could not be linked. Exiting \n"); + return -1; + } + } else { + if (!gst_element_link(queue2, sink)) { + g_printerr("Elements could not be linked. Exiting \n"); + return -1; + } + } + } else { + if (!gst_element_link (queue2, sink)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + } + + sink_pad = gst_element_get_static_pad (queue1, "sink"); + tee_msg_pad = gst_element_get_request_pad(tee, "src_%u"); + tee_render_pad = gst_element_get_request_pad(tee, "src_%u"); + + if (!tee_msg_pad || !tee_render_pad) { + g_printerr("Unable to request pads \n"); + return -1; + } + + if (gst_pad_link(tee_msg_pad, sink_pad) != GST_PAD_LINK_OK) { + g_printerr("Unable to link tee and message converter. \n"); + gst_object_unref(sink_pad); + return -1; + } + + gst_object_unref(sink_pad); + + sink_pad = gst_element_get_static_pad(queue2, "sink"); + if (gst_pad_link(tee_render_pad, sink_pad) != GST_PAD_LINK_OK) { + g_printerr("Unable to link tee and render pad. \n"); + gst_object_unref (sink_pad); + return -1; + } + + gst_object_unref(sink_pad); + + // Adding a probe to get informed of the meta data generated. + // We add a probe to the sink pad of the OSD element since by + // that time the buffer would have had got all the metadata + + osd_sink_pad = gst_element_get_static_pad (nvosd, "sink"); + if (!osd_sink_pad) { + g_print("Unable to get sink pad\n"); + } else { + if (msg2p_meta == 0) { + gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + osd_sink_pad_buffer_probe, NULL, NULL); + } + } + gst_object_unref(osd_sink_pad); + + // Set pipeline to playing state + if (argc > 1 && IS_YAML(argv[1])) { + g_print ("Using file %s \n", argv[1]); + } else { + g_print("Now playing %s\n", input_file); + } + gst_element_set_state(pipeline, GST_STATE_PLAYING); + + // Wait till the pipeline encounters an error or End of Stream (EOS) + g_print("Running ...\n"); + g_main_loop_run(loop); + + // Out of the main loop. Perform clean up + g_print ("Returned, stopped playback\n"); + + g_free (cfg_file); + g_free(input_file); + g_free(topic); + g_free(conn_str); + g_free(proto_lib); + + // Release the request pads from tee and unfer them + gst_element_release_request_pad(tee, tee_msg_pad); + gst_element_release_request_pad(tee, tee_render_pad); + gst_object_unref(tee_msg_pad); + gst_object_unref(tee_render_pad); + + gst_element_set_state(pipeline, GST_STATE_NULL); + g_print("Deleting pipeline \n"); + gst_object_unref(GST_OBJECT(pipeline)); + g_source_remove(bus_watch_id); + g_main_loop_unref(loop); + return 0; +} \ No newline at end of file diff --git a/pyservicemaker_sample_apps/README.md b/pyservicemaker_sample_apps/README.md new file mode 100644 index 0000000..cfe8c53 --- /dev/null +++ b/pyservicemaker_sample_apps/README.md @@ -0,0 +1,14 @@ +## Introduction +The apps in this directory are additional samples demonstrating usage of the Python API for DeepStream Service Maker, either by flow API or by pipeline API. See the [Python Service Maker documentation](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_service_maker_python.html) for details. + +Other sample reference apps for pyservicemaker can be found at `/opt/nvidia/deepstream/deepstream/service-maker/sources/apps/python/`. Pysm sample apps using TAO pre-trained models can be found in the [deepstream_tao_apps](https://github.com/NVIDIA-AI-IOT/deepstream_tao_apps/tree/master/pysm-apps) repo. + +## Prerequisites +* torchvision +``` +pip3 install torchvision +``` +* pyyaml +``` +pip3 install pyyaml +``` \ No newline at end of file diff --git a/pyservicemaker_sample_apps/flow_api/deepstream_nvdsanalytics_test_app/README.md b/pyservicemaker_sample_apps/flow_api/deepstream_nvdsanalytics_test_app/README.md new file mode 100644 index 0000000..306af9b --- /dev/null +++ b/pyservicemaker_sample_apps/flow_api/deepstream_nvdsanalytics_test_app/README.md @@ -0,0 +1,17 @@ +## Purpose + +The sample app demonstrates how to simplify [deepstream_nvds_analytics_test_app](../../pipeline_api/deepstream_nvdsanalytics_test_app) +using Flow API. +Flow APIs effectively abstract away the underlying pipeline details, allowing +developers to focus solely on the goals of their specific tasks in a pythonic style. + +## Usage +``` +$ python3 deepstream_nvdsanalytics.py [uri2] ... [uriN] +``` + +For URI(s) with special characters like @,& etc, you need to pass the uri within quotes + +``` +$ python3 deepstream_nvdsanalytics.py 'rtsp://user@ip/cam/realmonitor?channel=1&subtype=0' 'rtsp://user@ip/cam/realmonitor?channel=1&subtype=0' +``` \ No newline at end of file diff --git a/pyservicemaker_sample_apps/flow_api/deepstream_nvdsanalytics_test_app/deepstream_nvdsanalytics.py b/pyservicemaker_sample_apps/flow_api/deepstream_nvdsanalytics_test_app/deepstream_nvdsanalytics.py new file mode 100644 index 0000000..28c7944 --- /dev/null +++ b/pyservicemaker_sample_apps/flow_api/deepstream_nvdsanalytics_test_app/deepstream_nvdsanalytics.py @@ -0,0 +1,100 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### + +from pyservicemaker import Pipeline, Flow, BatchMetadataOperator, Probe, osd +from multiprocessing import Process +import sys +import platform +import os + +PIPELINE_NAME = "deepstream-nvdsanalytics-test" +CONFIG_FILE_PATH = "/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-nvdsanalytics-test/nvdsanalytics_pgie_config.txt" +ANALYTICS_CONFIG_FILE_PATH = "/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-nvdsanalytics-test/config_nvdsanalytics.txt" +TRACKER_LL_CONFIG_FILE = "/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml" +TRACKER_LL_LIB_FILE = "/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so" +BATCHED_PUSH_TIMEOUT = 33000 +MUXER_WIDTH = 1920 +MUXER_HEIGHT = 1080 +TILER_WIDTH = 1280 +TILER_HEIGHT = 720 + +class ObjectCounterMarker(BatchMetadataOperator): + def handle_metadata(self, batch_meta): + for frame_meta in batch_meta.frame_items: + vehicle_count = 0 + person_count = 0 + for object_meta in frame_meta.object_items: + class_id = object_meta.class_id + if class_id == 0: + vehicle_count += 1 + elif class_id == 2: + person_count += 1 + for user_meta in object_meta.nvdsanalytics_obj_items: + nvdsanalytics_obj_info = user_meta.as_nvdsanalytics_obj() + print("Object {0} moving in direction: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.dir_status)) + print("Object {0} line crossing status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.lc_status)) + print("Object {0} overcrowding status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.oc_status)) + print("Object {0} ROI status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.roi_status)) + print("Object {0} status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.obj_status)) + print("Object {0} unique ID: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.unique_id)) + for user_meta in frame_meta.nvdsanalytics_frame_items: + nvdsanalytics_frame_meta = user_meta.as_nvdsanalytics_frame() + print("Frame {0} overcrowding status: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.oc_status)) + print("Frame {0} object in ROI count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_in_roi_cnt)) + print("Frame {0} object line crossing current count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_lc_curr_cnt)) + print("Frame {0} object line crossing cumulative count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_lc_cum_cnt)) + print("Frame {0} unique ID: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.unique_id)) + print("Frame {0} object count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_cnt)) + print(f"Object Counter: Pad Idx={frame_meta.pad_index}," + f"Frame Number={frame_meta.frame_number}," + f"Vehicle Count={vehicle_count}, Person Count={person_count}") + display_text = f"Person={person_count},Vehicle={vehicle_count}" + display_meta = batch_meta.acquire_display_meta() + text = osd.Text() + text.display_text = display_text.encode('ascii') + text.x_offset = 10 + text.y_offset = 12 + text.font.name = osd.FontFamily.Serif + text.font.size = 12 + text.font.color = osd.Color(1.0, 1.0, 1.0, 1.0) + text.set_bg_color = True + text.bg_color = osd.Color(0.0, 0.0, 0.0, 1.0) + display_meta.add_text(text) + frame_meta.append(display_meta) + +def deepstream_nvdsanalytics_test_app(stream_file_path_list): + pipeline = Pipeline("deepstream-nvdsanalytics-test") + flow = Flow(pipeline).batch_capture(stream_file_path_list).infer(CONFIG_FILE_PATH) + flow = flow.track(ll_config_file=TRACKER_LL_CONFIG_FILE, ll_lib_file=TRACKER_LL_LIB_FILE) + flow = flow.analyze(ANALYTICS_CONFIG_FILE_PATH) + flow.attach(what=Probe("counter", ObjectCounterMarker())).render()() + +if __name__ == '__main__': + # Check input arguments + if len(sys.argv) < 2: + sys.stderr.write("usage: %s [uri2] ... [uriN]\n" % sys.argv[0]) + sys.exit(1) + + # Flow()() is a blocking call due to which the KeyboardInterrupt may not be processed immediately. + # we use Process from multiprocessing which runs the main function in a different process and processes KeyboardInterrupt immediately. + process = Process(target=deepstream_nvdsanalytics_test_app, args=(sys.argv[1:],)) + try: + process.start() + process.join() + except KeyboardInterrupt: + print("\nCtrl+C detected. Terminating process...") + process.terminate() \ No newline at end of file diff --git a/pyservicemaker_sample_apps/pipeline_api/deepstream_nvdsanalytics_test_app/README.md b/pyservicemaker_sample_apps/pipeline_api/deepstream_nvdsanalytics_test_app/README.md new file mode 100644 index 0000000..8df9fe2 --- /dev/null +++ b/pyservicemaker_sample_apps/pipeline_api/deepstream_nvdsanalytics_test_app/README.md @@ -0,0 +1,31 @@ +## Introduction + +This document describes the sample deepstream_nvdsanalytics_test application. + +This sample builds on top of the service-maker deepstream_test1 sample at at `/opt/nvidia/deepstream/deepstream/service-maker/sources/apps/python/pipeline_api/deepstream_test1_app` to demonstrate how to: + +* Use multiple sources in the pipeline. +* Use a uridecodebin so that any type of input (e.g. RTSP/File), any GStreamer + supported container format, and any codec can be used as input. +* Configure the stream-muxer to generate a batch of frames and infer on the + batch for better resource utilization. +* Perform analytics on metadata attached by nvinfer and nvtracker using nvdsanalytics plugin +* Extract the stream metadata, which contains useful information about the + frames in the batched buffer. Extract the analytics object level and frame level metadata, + which contains information about region of interest filtering, overcrowding detection, direction + detection, and line crossing. + +Refer to the service-maker deepstream_test1 sample documentation for an example of a +single-stream inference, bounding-box overlay, and rendering. + +## Usage + +Run with the uri(s). + +``` +$ python3 deepstream_nvdsanalytics.py [uri2] ... [uriN] +e.g. +$ python3 deepstream_nvdsanalytics.py file:///home/ubuntu/video1.mp4 file:///home/ubuntu/video2.mp4 +For URI(s) with special characters like @,& etc, you need to pass the uri within quotes +$ python3 deepstream_nvdsanalytics.py 'rtsp://user@ip/cam/realmonitor?channel=1&subtype=0' 'rtsp://user@ip/cam/realmonitor?channel=1&subtype=0' +``` diff --git a/pyservicemaker_sample_apps/pipeline_api/deepstream_nvdsanalytics_test_app/deepstream_nvdsanalytics.py b/pyservicemaker_sample_apps/pipeline_api/deepstream_nvdsanalytics_test_app/deepstream_nvdsanalytics.py new file mode 100644 index 0000000..dd2e050 --- /dev/null +++ b/pyservicemaker_sample_apps/pipeline_api/deepstream_nvdsanalytics_test_app/deepstream_nvdsanalytics.py @@ -0,0 +1,110 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### + +from pyservicemaker import Pipeline, Probe, BatchMetadataOperator, osd +from multiprocessing import Process +import sys +import platform +import os + +PIPELINE_NAME = "deepstream-nvdsanalytics-test" +CONFIG_FILE_PATH = "/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-nvdsanalytics-test/nvdsanalytics_pgie_config.txt" +ANALYTICS_CONFIG_FILE_PATH = "/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-nvdsanalytics-test/config_nvdsanalytics.txt" +TRACKER_LL_CONFIG_FILE = "/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml" +TRACKER_LL_LIB_FILE = "/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so" +BATCHED_PUSH_TIMEOUT = 33000 +MUXER_WIDTH = 1920 +MUXER_HEIGHT = 1080 +TILER_WIDTH = 1280 +TILER_HEIGHT = 720 + +class ObjectCounterMarker(BatchMetadataOperator): + def handle_metadata(self, batch_meta): + for frame_meta in batch_meta.frame_items: + vehicle_count = 0 + person_count = 0 + for object_meta in frame_meta.object_items: + class_id = object_meta.class_id + if class_id == 0: + vehicle_count += 1 + elif class_id == 2: + person_count += 1 + for user_meta in object_meta.nvdsanalytics_obj_items: + nvdsanalytics_obj_info = user_meta.as_nvdsanalytics_obj() + print("Object {0} moving in direction: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.dir_status)) + print("Object {0} line crossing status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.lc_status)) + print("Object {0} overcrowding status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.oc_status)) + print("Object {0} ROI status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.roi_status)) + print("Object {0} status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.obj_status)) + print("Object {0} unique ID: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.unique_id)) + for user_meta in frame_meta.nvdsanalytics_frame_items: + nvdsanalytics_frame_meta = user_meta.as_nvdsanalytics_frame() + print("Frame {0} overcrowding status: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.oc_status)) + print("Frame {0} object in ROI count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_in_roi_cnt)) + print("Frame {0} object line crossing current count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_lc_curr_cnt)) + print("Frame {0} object line crossing cumulative count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_lc_cum_cnt)) + print("Frame {0} unique ID: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.unique_id)) + print("Frame {0} object count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_cnt)) + print(f"Object Counter: Pad Idx={frame_meta.pad_index}," + f"Frame Number={frame_meta.frame_number}," + f"Vehicle Count={vehicle_count}, Person Count={person_count}") + display_text = f"Person={person_count},Vehicle={vehicle_count}" + display_meta = batch_meta.acquire_display_meta() + text = osd.Text() + text.display_text = display_text.encode('ascii') + text.x_offset = 10 + text.y_offset = 12 + text.font.name = osd.FontFamily.Serif + text.font.size = 12 + text.font.color = osd.Color(1.0, 1.0, 1.0, 1.0) + text.set_bg_color = True + text.bg_color = osd.Color(0.0, 0.0, 0.0, 1.0) + display_meta.add_text(text) + frame_meta.append(display_meta) + +def main(file_path): + if isinstance(file_path, list): + file_list = file_path if isinstance(file_path, list) else [file_path] + pipeline = Pipeline(PIPELINE_NAME) + pipeline.add("nvstreammux", "mux", {"batch-size": len(file_list), "batched-push-timeout": BATCHED_PUSH_TIMEOUT, "width": MUXER_WIDTH, "height": MUXER_HEIGHT, "compute-hw": 1, "nvbuf-memory-type": 2}) + for i, file in enumerate(file_list): + pipeline.add("uridecodebin", f"src_{i}", {"uri": file}) + pipeline.link((f"src_{i}", "mux"), ("", "sink_%u")) + pipeline.add("nvinfer", "infer", {"config-file-path": CONFIG_FILE_PATH, "batch-size": len(file_list)}) + pipeline.add("nvtracker", "tracker", {"ll-config-file": TRACKER_LL_CONFIG_FILE, "ll-lib-file": TRACKER_LL_LIB_FILE}) + pipeline.add("nvdsanalytics", "analytics", {"config-file": ANALYTICS_CONFIG_FILE_PATH}) + pipeline.add("nvmultistreamtiler", "tiler", {"width": TILER_WIDTH, "height": TILER_HEIGHT}) + pipeline.add("nvosdbin", "osd").add("nv3dsink" if platform.processor() == "aarch64" else "nveglglessink", "sink") + pipeline.link("mux", "infer", "tracker", "analytics", "tiler", "osd", "sink") + pipeline.attach("tiler", Probe("counter", ObjectCounterMarker())) + pipeline.start().wait() + +if __name__ == '__main__': + # Check input arguments + if len(sys.argv) < 2: + sys.stderr.write("usage: %s [uri2] ... [uriN]\n" % sys.argv[0]) + sys.exit(1) + + # pipeline.wait() in the main function is a blocking call due to which the KeyboardInterrupt may not be processed immediately. + # we use Process from multiprocessing which runs the main function in a different process and processes KeyboardInterrupt immediately. + process = Process(target=main, args=(sys.argv[1:],)) + try: + process.start() + process.join() + except KeyboardInterrupt: + print("\nCtrl+C detected. Terminating process...") + process.terminate() diff --git a/runtime_source_add_delete/Makefile b/runtime_source_add_delete/Makefile new file mode 100755 index 0000000..d8e35c6 --- /dev/null +++ b/runtime_source_add_delete/Makefile @@ -0,0 +1,61 @@ +################################################################################# +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +APP:= deepstream-test-rt-src-add-del + +TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -) + +DS_SDK_ROOT:=/opt/nvidia/deepstream/deepstream + +LIB_INSTALL_DIR?=$(DS_SDK_ROOT)/lib/ + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= -I$(DS_SDK_ROOT)/sources/includes \ + -I /usr/local/cuda-$(CUDA_VER)/include + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) + +LIBS:= $(shell pkg-config --libs $(PKGS)) + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_helper -lm -lnvdsgst_meta \ + -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart \ + -lcuda -Wl,-rpath,$(LIB_INSTALL_DIR) + +all: $(APP) + +.o: .c $(INCS) Makefile + $(CC) -c $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +clean: + rm -rf $(OBJS) $(APP) + + diff --git a/runtime_source_add_delete/README.md b/runtime_source_add_delete/README.md new file mode 100755 index 0000000..8a3cd37 --- /dev/null +++ b/runtime_source_add_delete/README.md @@ -0,0 +1,45 @@ +# RUNTIME SOURCE ADDITION DELETION REFERENCE APP USING DEEPSTREAMSDK 9.0 + +## Introduction +The project contains Runtime source addition/deletion application to show the +capability of Deepstream SDK. + +## Prerequisites: +DeepStream SDK installed which is available at http://developer.nvidia.com/deepstream-sdk +Please follow instructions in the apps/sample_apps/deepstream-app/README on how +to install the prequisites for Deepstream SDK apps. + +## Getting Started + +- Preferably clone the app in + `/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/` + +- Edit all the inference models config files according to the location of the models to be used + +## Compilation Steps and Execution: +``` + $ Set CUDA_VER in the MakeFile as per platform. + For x86, CUDA_VER=13.1 + For Jetson, CUDA_VER=13.0 + $ sudo make + + $ ./deepstream-test-rt-src-add-del + $ ./deepstream-test-rt-src-add-del file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h265.mp4 0 nveglglessink 1 #dGPU - nveglglessink Jetson - nv3dsink + $ ./deepstream-test-rt-src-add-del rtsp://127.0.0.1/video 0 nveglglessink 1 #dGPU +``` + +The application demonstrates following pipeline for single source + +uridecodebin -> nvstreammux -> nvinfer -> nvtracker -> nvtiler -> nvvideoconvert -> nvdsosd -> displaysink + +- At runtime after a timeout a source will be added periodically. All the components + are reconfigured during addition/deletion +- After reaching of `MAX_NUM_SOURCES`, each source is deleted periodically till single + source is present in the pipeline +- The app exits, when final source End of Stream is reached or if the last source is deleted. +- filesink and nv3dsink (only Jetson) are also supported. + + + + + diff --git a/runtime_source_add_delete/deepstream_test_rt_src_add_del.c b/runtime_source_add_delete/deepstream_test_rt_src_add_del.c new file mode 100755 index 0000000..3d36867 --- /dev/null +++ b/runtime_source_add_delete/deepstream_test_rt_src_add_del.c @@ -0,0 +1,725 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "gstnvdsmeta.h" +#include "gst-nvmessage.h" +#include "nvdsmeta.h" +#include + +#define MAX_DISPLAY_LEN 64 + +#define PGIE_CLASS_ID_VEHICLE 0 +#define PGIE_CLASS_ID_PERSON 2 +#define SET_GPU_ID(object, gpu_id) g_object_set (G_OBJECT (object), "gpu-id", gpu_id, NULL); +#define SET_MEMORY(object, mem_id) g_object_set (G_OBJECT (object), "nvbuf-memory-type", mem_id, NULL); + +GMainLoop *loop = NULL; +/* The muxer output resolution must be set if the input streams will be of + * different resolution. The muxer will scale all the input frames to this + * resolution. */ +#define MUXER_OUTPUT_WIDTH 1920 +#define MUXER_OUTPUT_HEIGHT 1080 + +#define TILED_OUTPUT_WIDTH 1280 +#define TILED_OUTPUT_HEIGHT 720 +#define GPU_ID 0 +#define MAX_NUM_SOURCES 4 +#define PGIE_CONFIG_FILE "dstest_pgie_config.txt" +#define TRACKER_CONFIG_FILE "dstest_tracker_config.txt" +#define SGIE1_CONFIG_FILE "dstest_sgie1_config.txt" +#define SGIE2_CONFIG_FILE "dstest_sgie2_config.txt" + + +#define CONFIG_GPU_ID "gpu-id" +#define CONFIG_GROUP_TRACKER "tracker" +#define CONFIG_GROUP_TRACKER_WIDTH "tracker-width" +#define CONFIG_GROUP_TRACKER_HEIGHT "tracker-height" +#define CONFIG_GROUP_TRACKER_LL_CONFIG_FILE "ll-config-file" +#define CONFIG_GROUP_TRACKER_LL_LIB_FILE "ll-lib-file" +#define CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS "enable-batch-process" + +gint g_num_sources = 0; +gint g_source_id_list[MAX_NUM_SOURCES]; +gboolean g_eos_list[MAX_NUM_SOURCES]; +gboolean g_source_enabled[MAX_NUM_SOURCES]; +GstElement **g_source_bin_list = NULL; +GMutex eos_lock; +gboolean g_run_forever = FALSE; + +/* Assuming Resnet 10 model packaged in DS SDK */ +gchar pgie_classes_str[4][32] = { "Vehicle", "TwoWheeler", "Person", + "Roadsign" +}; + +GstElement *pipeline = NULL, *streammux = NULL, *sink = NULL, *pgie = NULL, + *sgie1 = NULL, *sgie2 = NULL, + *nvvideoconvert = NULL, *nvosd = NULL, *tiler = NULL, *tracker = NULL, *queue = NULL; + +gchar *uri = NULL; + +static gboolean add_sources (gpointer data); + +static void +decodebin_child_added (GstChildProxy * child_proxy, GObject * object, + gchar * name, gpointer user_data) +{ + g_print ("decodebin child added %s\n", name); + if (g_strrstr (name, "decodebin") == name) { + g_signal_connect (G_OBJECT (object), "child-added", + G_CALLBACK (decodebin_child_added), user_data); + } + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + if (g_strrstr (name, "nvv4l2decoder") == name) { + if(prop.integrated) { + g_object_set (object, "enable-max-performance", TRUE, NULL); + g_object_set (object, "bufapi-version", TRUE, NULL); + g_object_set (object, "drop-frame-interval", 0, NULL); + g_object_set (object, "num-extra-surfaces", 0, NULL); + } else { + g_object_set (object, "gpu-id", GPU_ID, NULL); + } + } +} + +static gchar * +get_absolute_file_path (gchar *cfg_file_path, gchar *file_path) +{ + gchar abs_cfg_path[PATH_MAX + 1]; + gchar *abs_file_path; + gchar *delim; + + if (file_path && file_path[0] == '/') { + return file_path; + } + + if (!realpath (cfg_file_path, abs_cfg_path)) { + g_free (file_path); + return NULL; + } + + /* Return absolute path of config file if file_path is NULL. */ + if (!file_path) { + abs_file_path = g_strdup (abs_cfg_path); + return abs_file_path; + } + + delim = g_strrstr (abs_cfg_path, "/"); + *(delim + 1) = '\0'; + + abs_file_path = g_strconcat (abs_cfg_path, file_path, NULL); + g_free (file_path); + + return abs_file_path; +} + + +static void +cb_newpad (GstElement * decodebin, GstPad * pad, gpointer data) +{ + GstCaps *caps = gst_pad_query_caps (pad, NULL); + const GstStructure *str = gst_caps_get_structure (caps, 0); + const gchar *name = gst_structure_get_name (str); + + g_print ("decodebin new pad %s\n", name); + if (!strncmp (name, "video", 5)) { + gint source_id = (*(gint *) data); + gchar pad_name[16] = { 0 }; + GstPad *sinkpad = NULL; + g_snprintf (pad_name, 15, "sink_%u", source_id); + sinkpad = gst_element_request_pad_simple (streammux, pad_name); + if (gst_pad_link (pad, sinkpad) != GST_PAD_LINK_OK) { + g_print ("Failed to link decodebin to pipeline\n"); + } else { + g_print ("Decodebin linked to pipeline\n"); + } + gst_object_unref (sinkpad); + } +} + +static GstElement * +create_uridecode_bin (guint index, gchar * filename) +{ + GstElement *bin = NULL; + gchar bin_name[16] = { }; + + g_print ("creating uridecodebin for [%s]\n", filename); + g_source_id_list[index] = index; + g_snprintf (bin_name, 15, "source-bin-%02d", index); + bin = gst_element_factory_make ("uridecodebin", bin_name); + g_object_set (G_OBJECT (bin), "uri", filename, NULL); + g_signal_connect (G_OBJECT (bin), "pad-added", + G_CALLBACK (cb_newpad), &g_source_id_list[index]); + g_signal_connect (G_OBJECT (bin), "child-added", + G_CALLBACK (decodebin_child_added), &g_source_id_list[index]); + g_source_enabled[index] = TRUE; + + return bin; +} + +static void +stop_release_source (gint source_id) +{ + GstStateChangeReturn state_return; + gchar pad_name[16]; + GstPad *sinkpad = NULL; + state_return = + gst_element_set_state (g_source_bin_list[source_id], GST_STATE_NULL); + switch (state_return) { + case GST_STATE_CHANGE_SUCCESS: + g_print ("STATE CHANGE SUCCESS\n\n"); + g_snprintf (pad_name, 15, "sink_%u", source_id); + sinkpad = gst_element_get_static_pad (streammux, pad_name); + gst_pad_send_event (sinkpad, gst_event_new_eos ()); + gst_pad_send_event (sinkpad, gst_event_new_flush_stop (FALSE)); + gst_element_release_request_pad (streammux, sinkpad); + g_print ("STATE CHANGE SUCCESS %p\n\n", sinkpad); + gst_object_unref (sinkpad); + gst_bin_remove (GST_BIN (pipeline), g_source_bin_list[source_id]); + source_id--; + g_num_sources--; + break; + case GST_STATE_CHANGE_FAILURE: + g_print ("STATE CHANGE FAILURE\n\n"); + break; + case GST_STATE_CHANGE_ASYNC: + g_print ("STATE CHANGE ASYNC\n\n"); + g_snprintf (pad_name, 15, "sink_%u", source_id); + sinkpad = gst_element_get_static_pad (streammux, pad_name); + gst_pad_send_event (sinkpad, gst_event_new_eos ()); + gst_pad_send_event (sinkpad, gst_event_new_flush_stop (FALSE)); + gst_element_release_request_pad (streammux, sinkpad); + g_print ("STATE CHANGE ASYNC %p\n\n", sinkpad); + gst_object_unref (sinkpad); + gst_bin_remove (GST_BIN (pipeline), g_source_bin_list[source_id]); + source_id--; + g_num_sources--; + break; + case GST_STATE_CHANGE_NO_PREROLL: + g_print ("STATE CHANGE NO PREROLL\n\n"); + break; + default: + break; + } + + +} + +static gboolean +delete_sources (gpointer data) +{ + gint source_id; + g_mutex_lock (&eos_lock); + for (source_id = 0; source_id < MAX_NUM_SOURCES; source_id++) { + if (g_eos_list[source_id] == TRUE && g_source_enabled[source_id] == TRUE) { + g_source_enabled[source_id] = FALSE; + stop_release_source (source_id); + } + } + g_mutex_unlock (&eos_lock); + + if (g_num_sources == 0) { + if (g_run_forever==FALSE){ + g_main_loop_quit (loop); + g_print ("All sources Stopped quitting\n"); + } + else { + g_timeout_add_seconds (15, add_sources, (gpointer) g_source_bin_list); + } + return FALSE; + } + + do { + source_id = rand () % MAX_NUM_SOURCES; + } while (!g_source_enabled[source_id]); + g_source_enabled[source_id] = FALSE; + g_print ("Calling Stop %d \n", source_id); + stop_release_source (source_id); + + if (g_num_sources == 0) { + if (g_run_forever==FALSE){ + g_main_loop_quit (loop); + g_print ("All sources Stopped quitting\n"); + } + else { + g_timeout_add_seconds (15, add_sources, (gpointer) g_source_bin_list); + } + return FALSE; + } + + return TRUE; +} + +static gboolean +add_sources (gpointer data) +{ + gint source_id = g_num_sources; + GstElement *source_bin; + GstStateChangeReturn state_return; + + do { + /* Generating random source id between 0 - MAX_NUM_SOURCES - 1, + * which has not been enabled + */ + source_id = rand () % MAX_NUM_SOURCES; + } while (g_source_enabled[source_id]); + g_source_enabled[source_id] = TRUE; + + g_print ("Calling Start %d \n", source_id); + source_bin = create_uridecode_bin (source_id, uri); + if (!source_bin) { + g_printerr ("Failed to create source bin. Exiting.\n"); + return -1; + } + g_source_bin_list[source_id] = source_bin; + gst_bin_add (GST_BIN (pipeline), source_bin); + state_return = + gst_element_set_state (g_source_bin_list[source_id], GST_STATE_PLAYING); + switch (state_return) { + case GST_STATE_CHANGE_SUCCESS: + g_print ("STATE CHANGE SUCCESS\n\n"); + source_id++; + break; + case GST_STATE_CHANGE_FAILURE: + g_print ("STATE CHANGE FAILURE\n\n"); + break; + case GST_STATE_CHANGE_ASYNC: + g_print ("STATE CHANGE ASYNC\n\n"); + state_return = + gst_element_get_state (g_source_bin_list[source_id], NULL, NULL, + GST_CLOCK_TIME_NONE); + source_id++; + break; + case GST_STATE_CHANGE_NO_PREROLL: + g_print ("STATE CHANGE NO PREROLL\n\n"); + break; + default: + break; + } + g_num_sources++; + + + if (g_num_sources == MAX_NUM_SOURCES) { + /* We have reached MAX_NUM_SOURCES to be added, no stop calling this function + * and enable calling delete sources + */ + g_timeout_add_seconds (5, delete_sources, (gpointer) g_source_bin_list); + return FALSE; + } + + return TRUE; +} + +static gboolean +bus_call (GstBus * bus, GstMessage * msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + if (g_run_forever==FALSE){ + g_print ("End of stream\n"); + g_main_loop_quit (loop); + } + break; + case GST_MESSAGE_WARNING: + { + gchar *debug; + GError *error; + gst_message_parse_warning (msg, &error, &debug); + g_printerr ("WARNING from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + g_free (debug); + g_printerr ("Warning: %s\n", error->message); + g_error_free (error); + break; + } + case GST_MESSAGE_ERROR: + { + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + case GST_MESSAGE_ELEMENT: + { + if (gst_nvmessage_is_stream_eos (msg)) { + guint stream_id; + if (gst_nvmessage_parse_stream_eos (msg, &stream_id)) { + g_print ("Got EOS from stream %d\n", stream_id); + g_mutex_lock (&eos_lock); + g_eos_list[stream_id] = TRUE; + g_mutex_unlock (&eos_lock); + } + } + break; + } + default: + break; + } + return TRUE; +} + + +/* Tracker config parsing */ + +#define CHECK_ERROR(error) \ + if (error) { \ + g_printerr ("Error while parsing config file: %s\n", error->message); \ + goto done; \ + } + +static gboolean +set_tracker_properties (GstElement *nvtracker) +{ + gboolean ret = FALSE; + GError *error = NULL; + gchar **keys = NULL; + gchar **key = NULL; + GKeyFile *key_file = g_key_file_new (); + + if (!g_key_file_load_from_file (key_file, TRACKER_CONFIG_FILE, G_KEY_FILE_NONE, + &error)) { + if (error) { + g_printerr ("Failed to load config file: %s\n", error->message); + g_error_free (error); + } else { + g_printerr ("Failed to load config file.\n"); + } + return FALSE; + } + + keys = g_key_file_get_keys (key_file, CONFIG_GROUP_TRACKER, NULL, &error); + CHECK_ERROR (error); + + for (key = keys; *key; key++) { + if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_WIDTH)) { + gint width = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_WIDTH, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "tracker-width", width, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_HEIGHT)) { + gint height = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_HEIGHT, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "tracker-height", height, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GPU_ID)) { + guint gpu_id = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GPU_ID, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "gpu_id", gpu_id, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_LL_CONFIG_FILE)) { + char* ll_config_file = get_absolute_file_path (TRACKER_CONFIG_FILE, + g_key_file_get_string (key_file, + CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_LL_CONFIG_FILE, &error)); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "ll-config-file", ll_config_file, NULL); + g_free(ll_config_file); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_LL_LIB_FILE)) { + char* ll_lib_file = get_absolute_file_path (TRACKER_CONFIG_FILE, + g_key_file_get_string (key_file, + CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_LL_LIB_FILE, &error)); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "ll-lib-file", ll_lib_file, NULL); + g_free(ll_lib_file); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS)) { + gboolean enable_batch_process = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "enable_batch_process", + enable_batch_process, NULL); + } else { + g_printerr ("Unknown key '%s' for group [%s]", *key, + CONFIG_GROUP_TRACKER); + } + } + + ret = TRUE; +done: + if (error) { + g_error_free (error); + } + if (keys) { + g_strfreev (keys); + } + if (key_file) { + g_key_file_free (key_file); + } + if (!ret) { + g_printerr ("%s failed", __func__); + } + return ret; +} + +int +main (int argc, char *argv[]) +{ + GstBus *bus = NULL; + guint bus_watch_id; + guint i, num_sources; + guint tiler_rows, tiler_columns; + guint pgie_batch_size; + + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + gboolean sync = TRUE; + gboolean display = TRUE; + + gboolean enc_hw_support = TRUE; + + if (prop.integrated) { + FILE* ptr; + char device_name[50]; + ptr = fopen("/proc/device-tree/model", "r"); + + if(ptr){ + while (fgets(device_name, 50, ptr) != NULL) { + if (strstr(device_name,"Orin") && (strstr(device_name,"Nano"))) + enc_hw_support = FALSE; + } + fclose(ptr); + } + } + + /* Check input arguments */ + if ((argc != 5)) { + g_printerr ("Usage: %s \n", argv[0]); + g_printerr (" : 0 or 1 \n"); + g_printerr (" : filesink (generates test.mkv) or nveglglessink (dGPU) or nv3dsink (Jetson)\n"); + g_printerr (" : 0 or 1 \n\n"); + g_printerr ("example: %s file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 0 filesink 1\n", argv[0]); + return -1; + } + if (!strcmp(argv[3],"filesink")){ + display = FALSE; + } + else if (!strcmp(argv[3],"nveglglessink") || !strcmp(argv[3],"nv3dsink") ){ + display = TRUE; + } + else { + g_printerr ("Error: set correct sink: filesink, nveglglessink or nv3dsink\n"); + return -1; + } + + num_sources = 1; + g_run_forever = atoi(argv[2]); + sync = atoi(argv[4]); + + /* Standard GStreamer initialization */ + gst_init (&argc, &argv); + loop = g_main_loop_new (NULL, FALSE); + + g_mutex_init (&eos_lock); + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + pipeline = gst_pipeline_new ("dstest-pipeline"); + + /* Use nvinfer to run inferencing on decoder's output, + * behaviour of inferencing is set through config file */ + streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); + g_object_set (G_OBJECT (streammux), "batched-push-timeout", 25000, NULL); + g_object_set (G_OBJECT (streammux), "batch-size", 30, NULL); + g_object_set (G_OBJECT (streammux), "drop-pipeline-eos", g_run_forever, NULL); + SET_GPU_ID (streammux, GPU_ID); + + if (!pipeline || !streammux) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), streammux); + g_object_set (G_OBJECT (streammux), "live-source", 1, NULL); + + g_source_bin_list = g_malloc0 (sizeof (GstElement *) * MAX_NUM_SOURCES); + uri = g_strdup (argv[1]); + for (i = 0; i < /*num_sources */ 1; i++) { + GstElement *source_bin = create_uridecode_bin (i, argv[i + 1]); + if (!source_bin) { + g_printerr ("Failed to create source bin. Exiting.\n"); + return -1; + } + g_source_bin_list[i] = source_bin; + gst_bin_add (GST_BIN (pipeline), source_bin); + } + + g_num_sources = num_sources; + + /* Use nvinfer to run inferencing on decoder's output, + * behaviour of inferencing is set through config file + */ + pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine"); + + /* Use nvtiler to stitch o/p from upstream components */ + tiler = gst_element_factory_make ("nvmultistreamtiler", "nvtiler"); + + /* Use convertor to convert from NV12 to RGBA as required by nvosd */ + nvvideoconvert = + gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); + + /* Create OSD to draw on the converted RGBA buffer */ + nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); + + tracker = gst_element_factory_make ("nvtracker", "nvtracker"); + + sgie1 = gst_element_factory_make ("nvinfer", "secondary-nvinference-engine1"); + sgie2 = gst_element_factory_make ("nvinfer", "secondary-nvinference-engine2"); + queue = gst_element_factory_make ("queue", "queue"); + + if (display){ + /* Finally render the osd output */ + if (prop.integrated) { + sink = gst_element_factory_make ("nv3dsink", "nv3dsink"); + } else { +#ifdef __aarch64__ + sink = gst_element_factory_make ("nv3dsink", "nv3dsink"); +#else + sink = gst_element_factory_make ("nveglglessink", "nveglglessink"); +#endif + } + } + else { + sink = gst_element_factory_make ("nvvideoencfilesinkbin","sink"); + g_object_set (G_OBJECT(sink), "container", 2, "output-file", "test.mkv", NULL); + if (!enc_hw_support){ + g_object_set(G_OBJECT(sink), "enc-type", 1, NULL); + } + } + + if (!pgie || !sgie1 || !sgie2 || !tiler || !nvvideoconvert || !nvosd + || !sink || !tracker) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + + g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", + MUXER_OUTPUT_HEIGHT, NULL); + + /* Set all the necessary properties of the nvinfer element, + * the necessary ones are : */ + g_object_set (G_OBJECT (pgie), "config-file-path", PGIE_CONFIG_FILE, NULL); + g_object_set (G_OBJECT (sgie1), "config-file-path", SGIE1_CONFIG_FILE, NULL); + g_object_set (G_OBJECT (sgie2), "config-file-path", SGIE2_CONFIG_FILE, NULL); + + /* Set necessary properties of the tracker element. */ + if (!set_tracker_properties(tracker)) { + g_printerr ("Failed to set tracker properties. Exiting.\n"); + return -1; + } + + /* Set all the necessary properties of the nvinfer element, + * the necessary ones are : */ + g_object_get (G_OBJECT (pgie), "batch-size", &pgie_batch_size, NULL); + if (pgie_batch_size < MAX_NUM_SOURCES) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + pgie_batch_size, num_sources); + g_object_set (G_OBJECT (pgie), "batch-size", MAX_NUM_SOURCES, NULL); + } + + /* Set GPU ID of elements */ + SET_GPU_ID (pgie, GPU_ID); + SET_GPU_ID (sgie1, GPU_ID); + SET_GPU_ID (sgie2, GPU_ID); + + tiler_rows = (guint) sqrt (num_sources); + tiler_columns = (guint) ceil (1.0 * num_sources / tiler_rows); + /* we set the osd properties here */ + g_object_set (G_OBJECT (tiler), "rows", tiler_rows, "columns", tiler_columns, + "width", TILED_OUTPUT_WIDTH, "height", TILED_OUTPUT_HEIGHT, NULL); + SET_GPU_ID (tiler, GPU_ID); + SET_GPU_ID (nvvideoconvert, GPU_ID); + SET_GPU_ID (nvosd, GPU_ID); + if(!prop.integrated) { +#ifndef __aarch64__ + SET_GPU_ID (sink, GPU_ID); +#endif + } + + /* we add a message handler */ + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); + gst_object_unref (bus); + + /* Set up the pipeline */ + /* we add all elements into the pipeline */ + gst_bin_add_many (GST_BIN (pipeline), pgie, tracker, sgie1, sgie2, + tiler, nvvideoconvert, nvosd, queue, sink, NULL); + + /* we link the elements together */ + /* file-source -> h264-parser -> nvh264-decoder -> + * nvinfer -> nvvideoconvert -> nvosd -> video-renderer */ + if (!gst_element_link_many (streammux, pgie, tracker, sgie1, sgie2, queue, + tiler, nvvideoconvert, nvosd, sink, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + + g_object_set (G_OBJECT (sink), "sync", sync, "qos", FALSE, NULL); + + gst_element_set_state (pipeline, GST_STATE_PAUSED); + + /* Set the pipeline to "playing" state */ + g_print ("Now playing: %s\n", argv[1]); + if (gst_element_set_state (pipeline, + GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { + g_printerr ("Failed to set pipeline to playing. Exiting.\n"); + return -1; + } + //GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS (GST_BIN (pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "ds-app-playing"); + + /* Wait till pipeline encounters an error or EOS */ + g_print ("Running...\n"); + g_timeout_add_seconds (15, add_sources, (gpointer) g_source_bin_list); + g_main_loop_run (loop); + + /* Out of the main loop, clean up nicely */ + g_print ("Returned, stopping playback\n"); + gst_element_set_state (pipeline, GST_STATE_NULL); + g_print ("Deleting pipeline\n"); + gst_object_unref (GST_OBJECT (pipeline)); + g_source_remove (bus_watch_id); + g_main_loop_unref (loop); + g_free (g_source_bin_list); + g_free (uri); + g_mutex_clear (&eos_lock); + return 0; +} + diff --git a/runtime_source_add_delete/dstest_pgie_config.txt b/runtime_source_add_delete/dstest_pgie_config.txt new file mode 100755 index 0000000..d474c80 --- /dev/null +++ b/runtime_source_add_delete/dstest_pgie_config.txt @@ -0,0 +1,94 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path, +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.00392156862745098 +onnx-file=../../../../../samples/models/Primary_Detector/resnet18_trafficcamnet_pruned.onnx +model-engine-file=../../../../../samples/models/Primary_Detector/resnet18_trafficcamnet_pruned.onnx_b1_gpu0_fp16.engine +labelfile-path=../../../../../samples/models/Primary_Detector/labels.txt +int8-calib-file=../../../../../samples/models/Primary_Detector/cal_trt.bin +batch-size=30 +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=4 +interval=0 +gie-unique-id=1 +#parse-bbox-func-name=NvDsInferParseCustomResnet +#custom-lib-path=/path/to/libnvdsparsebbox.so +#enable-dbscan=1 +cluster-mode=2 + +[class-attrs-all] +pre-cluster-threshold=0.2 +topk=20 +nms-iou-threshold=0.5 +#minBoxes=3 +roi-top-offset=0 +roi-bottom-offset=0 +detected-min-w=0 +detected-min-h=0 +detected-max-w=0 +detected-max-h=0 + +## Per class configuration +#[class-attrs-2] +#pre-cluster-threshold=0.6 +#eps=0.5 +#group-threshold=3 +#roi-top-offset=20 +#roi-bottom-offset=10 +#detected-min-w=40 +#detected-min-h=40 +#detected-max-w=400 +#detected-max-h=800 diff --git a/runtime_source_add_delete/dstest_sgie1_config.txt b/runtime_source_add_delete/dstest_sgie1_config.txt new file mode 100755 index 0000000..ac2f1e7 --- /dev/null +++ b/runtime_source_add_delete/dstest_sgie1_config.txt @@ -0,0 +1,73 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path, +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=1 +onnx-file=../../../../../samples/models/Secondary_VehicleMake/resnet18_vehiclemakenet_pruned.onnx +model-engine-file=../../../../../samples/models/Secondary_VehicleMake/resnet18_vehiclemakenet_pruned.onnx_b16_gpu0_fp16.engine +labelfile-path=../../../../../samples/models/Secondary_VehicleMake/labels.txt +int8-calib-file=../../../../../samples/models/Secondary_VehicleMake/cal_trt.bin +batch-size=16 +# 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +input-object-min-width=64 +input-object-min-height=64 +process-mode=2 +model-color-format=1 +gpu-id=0 +gie-unique-id=2 +operate-on-gie-id=1 +operate-on-class-ids=0 +is-classifier=1 +classifier-async-mode=0 +classifier-threshold=0.51 +process-mode=2 diff --git a/runtime_source_add_delete/dstest_sgie2_config.txt b/runtime_source_add_delete/dstest_sgie2_config.txt new file mode 100755 index 0000000..2ea8e2f --- /dev/null +++ b/runtime_source_add_delete/dstest_sgie2_config.txt @@ -0,0 +1,73 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path, +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=1 +onnx-file=../../../../../samples/models/Secondary_VehicleTypes/resnet18_vehicletypenet_pruned.onnx +model-engine-file=../../../../../samples/models/Secondary_VehicleTypes/resnet18_vehicletypenet_pruned.onnx_b16_gpu0_fp16.engine +labelfile-path=../../../../../samples/models/Secondary_VehicleTypes/labels.txt +int8-calib-file=../../../../../samples/models/Secondary_VehicleTypes/cal_trt.bin +batch-size=16 +# 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +input-object-min-width=64 +input-object-min-height=64 +process-mode=2 +model-color-format=1 +gpu-id=0 +gie-unique-id=3 +operate-on-gie-id=1 +operate-on-class-ids=0 +is-classifier=1 +classifier-async-mode=0 +classifier-threshold=0.51 +process-mode=2 diff --git a/runtime_source_add_delete/dstest_tracker_config.txt b/runtime_source_add_delete/dstest_tracker_config.txt new file mode 100755 index 0000000..324eada --- /dev/null +++ b/runtime_source_add_delete/dstest_tracker_config.txt @@ -0,0 +1,38 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Mandatory properties for the tracker: +# tracker-width +# tracker-height: needs to be multiple of 32 for NvDCF +# gpu-id +# ll-lib-file: path to low-level tracker lib +# ll-config-file: required for NvDCF, optional for KLT and IOU +# +[tracker] +# For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively +tracker-width=640 +tracker-height=384 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +# ll-config-file required to set different tracker types +# ll-config-file=config_tracker_IOU.yml +ll-config-file=config_tracker_NvDCF_perf.yml +# ll-config-file=config_tracker_NvDCF_accuracy.yml +# ll-config-file=config_tracker_DeepSORT.yml +gpu-id=0 +enable-batch-process=1 +enable-past-frame=1 +display-tracking-id=1 diff --git a/runtime_source_add_delete/tracker_config.yml b/runtime_source_add_delete/tracker_config.yml new file mode 100755 index 0000000..4c03f7a --- /dev/null +++ b/runtime_source_add_delete/tracker_config.yml @@ -0,0 +1,36 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +%YAML:1.0 + +NvDCF: + maxTargetsPerStream: 30 # Max number of targets to track per stream. Recommended to set >10. Note: this value should account for the targets being tracked in shadow mode as well. Max value depends on the GPU memory capacity + + filterLr: 0.11 # learning rate for DCF filter in exponential moving average. Valid Range: [0.0, 1.0] + gaussianSigma: 0.75 # Standard deviation for Gaussian for desired response when creating DCF filter + + minDetectorConfidence: 0.0 # If the confidence of a detector bbox is lower than this, then it won't be considered for tracking + minTrackerConfidence: 0.6 # If the confidence of an object tracker is lower than this on the fly, then it will be tracked in shadow mode. Valid Range: [0.0, 1.0] + + featureImgSizeLevel: 1 # Size of a feature image. Valid range: {1, 2, 3, 4, 5}, from the smallest to the largest + SearchRegionPaddingScale: 3 # Search region size. Determines how large the search region should be scaled from the target bbox. Valid range: {1, 2, 3}, from the smallest to the largest + + maxShadowTrackingAge: 9 # Max length of shadow tracking (the shadow tracking age is incremented when (1) there's detector input yet no match or (2) tracker confidence is lower than minTrackerConfidence). Once reached, the tracker will be terminated. + probationAge: 0 # Once the tracker age (incremented at every frame) reaches this, the tracker is considered to be valid + earlyTerminationAge: 0 # Early termination age (in terms of shadow tracking age) during the probation period + + minVisibiilty4Tracking: 0.5 # If the visibility of the bbox of a tracker gets lower, then it will be terminated diff --git a/sources/apps/TRT-yolo/Makefile b/sources/apps/TRT-yolo/Makefile deleted file mode 100644 index 095c6ea..0000000 --- a/sources/apps/TRT-yolo/Makefile +++ /dev/null @@ -1,64 +0,0 @@ -# MIT License - -# Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -CXX := g++ -CUDA_VER:=9.2 -APP:= TRT-yolo-app -YOLO_LIB:= ../../gst-yoloplugin/yoloplugin_lib - -OBJS = $(wildcard $(YOLO_LIB)/build/*.o) -DEPS:= $(wildcard $(YOLO_LIB)/*.cpp) -DEPS+= $(wildcard $(YOLO_LIB)/*.h) - -CONFIG :=../../../Makefile.config -ifeq ($(wildcard $(CONFIG)),) -$(error $(CONFIG) missing.) -endif -include $(CONFIG) - -CXXFLAGS:= -O2 -std=c++11 -lstdc++fs -Wall -Wunused-function -Wunused-variable -I $(YOLO_LIB) - -LIBS:= -L "$(TENSORRT_INSTALL_DIR)/lib" -lnvinfer -lnvinfer_plugin -Wl,-rpath="$(TENSORRT_INSTALL_DIR)/lib" \ - -L "/usr/local/cuda-$(CUDA_VER)/lib64" -lcudart -lcublas -lcurand -Wl,-rpath="/usr/local/cuda-$(CUDA_VER)/lib64" \ - -L "$(OPENCV_INSTALL_DIR)/lib" -lopencv_core -lopencv_imgproc -lopencv_imgcodecs -lopencv_highgui -lopencv_dnn -Wl,-rpath="$(OPENCV_INSTALL_DIR)/lib" \ - -L "/usr/lib/x86_64-linux-gnu" -lgflags - -.PHONY: all deps install clean - -all: $(APP) - -$(APP): deps - @echo Building $(APP) - $(CXX) -o $(APP) TRT-yolo-app.cpp $(OBJS) $(CXXFLAGS) $(LIBS) - -deps: $(DEPS) - @echo "Building yoloplugin_lib" - $(MAKE) -C $(YOLO_LIB) dirs - $(MAKE) -C $(YOLO_LIB) deps CXX="$(CXX)" CUDA_VER="$(CUDA_VER)" TENSORRT_INSTALL_DIR="$(TENSORRT_INSTALL_DIR)" OPENCV_INSTALL_DIR="$(OPENCV_INSTALL_DIR)" - -install: $(APP) - @echo Installing $(APP) - cp -rv $(APP) /usr/bin/ - -clean: - rm -rf $(APP) - $(MAKE) -C $(YOLO_LIB) clean \ No newline at end of file diff --git a/sources/apps/TRT-yolo/TRT-yolo-app.cpp b/sources/apps/TRT-yolo/TRT-yolo-app.cpp deleted file mode 100644 index db9ef33..0000000 --- a/sources/apps/TRT-yolo/TRT-yolo-app.cpp +++ /dev/null @@ -1,137 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#include "ds_image.h" -#include "network_config.h" -#include "trt_utils.h" -#include "yolo.h" - -#ifdef MODEL_V2 -#include "yolov2.h" -#endif - -#ifdef MODEL_V3 -#include "yolov3.h" -#endif - -#include -#include -#include -#include - -DEFINE_bool(decode, true, "Decode the detections"); -DEFINE_int32(batch_size, 1, "Batch size for the inference engine."); -DEFINE_uint64(seed, std::time(0), "Seed for the random number generator"); - -int main(int argc, char** argv) -{ - srand(unsigned(FLAGS_seed)); - ::google::ParseCommandLineFlags(&argc, &argv, true); - - std::unique_ptr inferNet{nullptr}; - -#ifdef MODEL_V2 - inferNet = std::unique_ptr{new YoloV2(FLAGS_batch_size)}; -#endif - -#ifdef MODEL_V3 - inferNet = std::unique_ptr{new YoloV3(FLAGS_batch_size)}; -#endif - - std::vector imageList = loadImageList(config::kTEST_IMAGES); - imageList.resize(static_cast(imageList.size() / FLAGS_batch_size) * FLAGS_batch_size); - std::random_shuffle(imageList.begin(), imageList.end(), [](int i) { return rand() % i; }); - std::cout << "Total number of images used for inference : " << imageList.size() << std::endl; - - std::vector dsImages(FLAGS_batch_size); - const int barWidth = 70; - double inferElapsed = 0; - int batchCount = imageList.size() / FLAGS_batch_size; - - // Batched inference loop - for (uint loopIdx = 0; loopIdx < imageList.size(); loopIdx += FLAGS_batch_size) - { - // Load a new batch - for (uint imageIdx = loopIdx; imageIdx < (loopIdx + FLAGS_batch_size); ++imageIdx) - { - dsImages.at(imageIdx - loopIdx) = DsImage(imageList.at(imageIdx), inferNet->getInputH(), - inferNet->getInputW()); - } - - cv::Mat trtInput = blobFromDsImages(dsImages, inferNet->getInputH(), inferNet->getInputW()); - struct timeval inferStart, inferEnd; - gettimeofday(&inferStart, NULL); - inferNet->doInference(trtInput.data); - gettimeofday(&inferEnd, NULL); - inferElapsed += ((inferEnd.tv_sec - inferStart.tv_sec) - + (inferEnd.tv_usec - inferStart.tv_usec) / 1000000.0) - * 1000 / FLAGS_batch_size; - - if (FLAGS_decode) - { - for (int imageIdx = 0; imageIdx < FLAGS_batch_size; ++imageIdx) - { - auto curImage = dsImages.at(imageIdx); - auto binfo = inferNet->decodeDetections(imageIdx, curImage.getImageHeight(), - curImage.getImageWidth()); - auto remaining = nonMaximumSuppression(inferNet->getNMSThresh(), binfo); - for (auto b : remaining) - { - if (inferNet->isPrintPredictions()) - { - printPredictions(b, inferNet->getClassName(b.label)); - } - curImage.addBBox(b, inferNet->getClassName(b.label)); - } - - if (config::kSAVE_DETECTIONS) - { - curImage.saveImageJPEG(config::kDETECTION_RESULTS_PATH); - } - } - } - - std::cout << "["; - int progress = ((loopIdx + FLAGS_batch_size) * 100) / imageList.size(); - progress = progress > 100 ? 100 : progress; - int pos = (barWidth * progress) / 100; - for (int i = 0; i < pos; ++i) - { - std::cout << "="; - } - if (pos < barWidth) std::cout << ">"; - for (int i = pos; i < barWidth; ++i) - { - std::cout << " "; - } - std::cout << "] " << progress << " %\r"; - std::cout.flush(); - } - std::cout << std::endl - << "Network Type : " << inferNet->getNetworkType() - << "Precision : " << config::kPRECISION << " Batch Size : " << FLAGS_batch_size - << " Inference time per image : " << inferElapsed / batchCount << " ms" << std::endl; - return 0; -} diff --git a/sources/apps/deepstream-yolo/Makefile b/sources/apps/deepstream-yolo/Makefile deleted file mode 100644 index 66ba546..0000000 --- a/sources/apps/deepstream-yolo/Makefile +++ /dev/null @@ -1,67 +0,0 @@ -# MIT License - -# Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -CXX := g++ - -APP:= deepstream-yolo-app - -SRCS:= $(wildcard *.cpp) - -INCS:= $(wildcard *.h) - -PKGS:= gstreamer-1.0 - -OBJS:= $(SRCS:.cpp=.o) - -CONFIG :=../../../Makefile.config -ifeq ($(wildcard $(CONFIG)),) -$(error $(CONFIG) missing.) -endif -include $(CONFIG) - -CXXFLAGS:= -I $(DEEPSTREAM_INSTALL_DIR)/sources/includes \ - -I /usr/include/gstreamer-1.0 - -CXXFLAGS+= -std=c++11 -Wunused-variable `pkg-config --cflags $(PKGS)` - -LIBS:= `pkg-config --libs $(PKGS)` - -.PHONY: all install clean - -all: $(APP) - -.o: .cpp $(INCS) Makefile - $(CXX) -c $@ $(CXXFLAGS) $< - -$(APP): $(OBJS) Makefile - @echo Building $(APP) - $(CXX) -o $(APP) $(OBJS) $(CXXFLAGS) $(LIBS) - -install: $(APP) - @echo Installing $(APP) - cp -rv $(APP) /usr/bin/ - -clean: - rm -rf $(OBJS) $(APP) - - diff --git a/sources/apps/deepstream-yolo/deepstream-yolo-app.cpp b/sources/apps/deepstream-yolo/deepstream-yolo-app.cpp deleted file mode 100644 index 59c33bf..0000000 --- a/sources/apps/deepstream-yolo/deepstream-yolo-app.cpp +++ /dev/null @@ -1,247 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#include -#include - -#include "gstnvdsmeta.h" - -/* As defined in the yolo plugins header*/ - -#define YOLO_UNIQUE_ID 15 - -gint frame_number = 0; - -/* osd_sink_pad_buffer_probe will extract metadata received on OSD sink pad - * and get a count of objects of interest */ - -static GstPadProbeReturn osd_sink_pad_buffer_probe(GstPad* pad, GstPadProbeInfo* info, - gpointer u_data) -{ - - GstMeta* gst_meta = NULL; - NvDsMeta* nvdsmeta = NULL; - gpointer state = NULL; - static GQuark _nvdsmeta_quark = 0; - GstBuffer* buf = (GstBuffer*) info->data; - NvDsFrameMeta* frame_meta = NULL; - guint num_rects = 0, rect_index = 0; - NvDsObjectParams* obj_meta = NULL; - guint car_count = 0; - guint person_count = 0; - guint bicycle_count = 0; - guint truck_count = 0; - - if (!_nvdsmeta_quark) _nvdsmeta_quark = g_quark_from_static_string(NVDS_META_STRING); - - while ((gst_meta = gst_buffer_iterate_meta(buf, &state))) - { - if (gst_meta_api_type_has_tag(gst_meta->info->api, _nvdsmeta_quark)) - { - - nvdsmeta = (NvDsMeta*) gst_meta; - - /* We are interested only in intercepting Meta of type - * "NVDS_META_FRAME_INFO" as they are from our infer elements. */ - if (nvdsmeta->meta_type == NVDS_META_FRAME_INFO) - { - frame_meta = (NvDsFrameMeta*) nvdsmeta->meta_data; - if (frame_meta == NULL) - { - g_print("NvDS Meta contained NULL meta \n"); - return GST_PAD_PROBE_OK; - } - - num_rects = frame_meta->num_rects; - - /* This means we have num_rects in frame_meta->obj_params. - * Now lets iterate through them and count the number of cars, - * trucks, persons and bicycles in each frame */ - - for (rect_index = 0; rect_index < num_rects; rect_index++) - { - obj_meta = (NvDsObjectParams*) &frame_meta->obj_params[rect_index]; - if (!g_strcmp0(obj_meta->attr_info[YOLO_UNIQUE_ID].attr_label, "car")) - car_count++; - else if (!g_strcmp0(obj_meta->attr_info[YOLO_UNIQUE_ID].attr_label, "person")) - person_count++; - else if (!g_strcmp0(obj_meta->attr_info[YOLO_UNIQUE_ID].attr_label, "bicycle")) - bicycle_count++; - else if (!g_strcmp0(obj_meta->attr_info[YOLO_UNIQUE_ID].attr_label, "truck")) - truck_count++; - } - } - } - } - g_print( - "Frame Number = %d Number of objects = %d " - "Car Count = %d Person Count = %d " - "Bicycle Count = %d Truck Count = %d \n", - frame_number, num_rects, car_count, person_count, bicycle_count, truck_count); - frame_number++; - - return GST_PAD_PROBE_OK; -} - -static gboolean bus_call(GstBus* bus, GstMessage* msg, gpointer data) -{ - GMainLoop* loop = (GMainLoop*) data; - switch (GST_MESSAGE_TYPE(msg)) - { - case GST_MESSAGE_EOS: - g_print("End of stream\n"); - g_main_loop_quit(loop); - break; - case GST_MESSAGE_ERROR: - { - gchar* debug; - GError* error; - gst_message_parse_error(msg, &error, &debug); - g_printerr("ERROR from element %s: %s\n", GST_OBJECT_NAME(msg->src), error->message); - g_free(debug); - g_printerr("Error: %s\n", error->message); - g_error_free(error); - g_main_loop_quit(loop); - break; - } - default: break; - } - return TRUE; -} - -int main(int argc, char* argv[]) -{ - GMainLoop* loop = NULL; - GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, *decoder = NULL, *sink = NULL, - *nvvidconv = NULL, *nvosd = NULL, *filter1 = NULL, *filter2 = NULL, *yolo = NULL; - GstBus* bus = NULL; - guint bus_watch_id; - GstCaps *caps1 = NULL, *caps2 = NULL; - gulong osd_probe_id = 0; - GstPad* osd_sink_pad = NULL; - - /* Check input arguments */ - if (argc != 2) - { - g_printerr("Usage: %s \n", argv[0]); - return -1; - } - - /* Standard GStreamer initialization */ - gst_init(&argc, &argv); - loop = g_main_loop_new(NULL, FALSE); - - /* Create gstreamer elements */ - /* Create Pipeline element that will form a connection of other elements */ - pipeline = gst_pipeline_new("dstest1-pipeline"); - - /* Source element for reading from the file */ - source = gst_element_factory_make("filesrc", "file-source"); - - /* Since the data format in the input file is elementary h264 stream, - * we need a h264parser */ - h264parser = gst_element_factory_make("h264parse", "h264-parser"); - - /* Use nvdec_h264 for hardware accelerated decode on GPU */ - decoder = gst_element_factory_make("nvdec_h264", "nvh264-decoder"); - - /* Use convertor to convert from NV12 to RGBA as required by nvosd and yolo plugins */ - nvvidconv = gst_element_factory_make("nvvidconv", "nvvideo-converter"); - - /* Use yolo to run inference instead of pgie */ - yolo = gst_element_factory_make("nvyolo", "yolo-inference-engine"); - - /* Create OSD to draw on the converted RGBA buffer */ - nvosd = gst_element_factory_make("nvosd", "nv-onscreendisplay"); - - /* Finally render the osd output */ - sink = gst_element_factory_make("nveglglessink", "nvvideo-renderer"); - - /* caps filter for nvvidconv to convert NV12 to RGBA as nvosd expects input - * in RGBA format */ - filter1 = gst_element_factory_make("capsfilter", "filter1"); - filter2 = gst_element_factory_make("capsfilter", "filter2"); - if (!pipeline || !source || !h264parser || !decoder || !filter1 || !nvvidconv || !filter2 - || !nvosd || !sink || !yolo) - { - g_printerr("One element could not be created. Exiting.\n"); - return -1; - } - - /* we set the input filename to the source element */ - g_object_set(G_OBJECT(source), "location", argv[1], NULL); - - /* we set the osd properties here */ - g_object_set(G_OBJECT(nvosd), "font-size", 15, NULL); - - /* we add a message handler */ - bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline)); - bus_watch_id = gst_bus_add_watch(bus, bus_call, loop); - gst_object_unref(bus); - - /* Set up the pipeline */ - /* we add all elements into the pipeline */ - gst_bin_add_many(GST_BIN(pipeline), source, h264parser, decoder, filter1, nvvidconv, filter2, - yolo, nvosd, sink, NULL); - caps1 = gst_caps_from_string("video/x-raw(memory:NVMM), format=NV12"); - g_object_set(G_OBJECT(filter1), "caps", caps1, NULL); - gst_caps_unref(caps1); - caps2 = gst_caps_from_string("video/x-raw(memory:NVMM), format=RGBA"); - g_object_set(G_OBJECT(filter2), "caps", caps2, NULL); - gst_caps_unref(caps2); - - /* we link the elements together */ - /* file-source -> h264-parser -> nvh264-decoder -> - * filter1 -> nvvidconv -> filter2 -> yolo -> nvosd -> video-renderer */ - gst_element_link_many(source, h264parser, decoder, filter1, nvvidconv, filter2, yolo, nvosd, - sink, NULL); - - /* Lets add probe to get informed of the meta data generated, we add probe to - * the sink pad of the osd element, since by that time, the buffer would have - * had got all the metadata. */ - osd_sink_pad = gst_element_get_static_pad(nvosd, "sink"); - if (!osd_sink_pad) - g_print("Unable to get sink pad\n"); - else - osd_probe_id = gst_pad_add_probe(osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, - osd_sink_pad_buffer_probe, NULL, NULL); - - /* Set the pipeline to "playing" state */ - g_print("Now playing: %s\n", argv[1]); - gst_element_set_state(pipeline, GST_STATE_PLAYING); - - /* Wait till pipeline encounters an error or EOS */ - g_print("Running...\n"); - g_main_loop_run(loop); - - /* Out of the main loop, clean up nicely */ - g_print("Returned, stopping playback\n"); - gst_element_set_state(pipeline, GST_STATE_NULL); - g_print("Deleting pipeline\n"); - gst_object_unref(GST_OBJECT(pipeline)); - g_source_remove(bus_watch_id); - g_main_loop_unref(loop); - return 0; -} diff --git a/sources/gst-yoloplugin/Makefile b/sources/gst-yoloplugin/Makefile deleted file mode 100644 index 5f3547d..0000000 --- a/sources/gst-yoloplugin/Makefile +++ /dev/null @@ -1,77 +0,0 @@ -# MIT License - -# Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -CUDA_VER:=9.2 -CXX:= g++ -SRCS:= gstyoloplugin.cpp -INCS:= $(wildcard *.h) -LIB:=libgstnvyolo.so - -CONFIG :=../../Makefile.config -ifeq ($(wildcard $(CONFIG)),) -$(error $(CONFIG) missing.) -endif -include $(CONFIG) - -SUBDIR := yoloplugin_lib -DEP:=$(SUBDIR)/libyoloplugin.a -DEP_FILES:=$(wildcard $(SUBDIR)/*.cpp) -DEP_FILES+=$(wildcard $(SUBDIR)/*.h) - -CXXFLAGS := -O2 -fPIC -std=c++11 -lstdc++fs -Wall -Wunused-function -Wunused-variable -INCS:= -I"$(TENSORRT_INSTALL_DIR)/include" \ - -I"/usr/local/cuda-$(CUDA_VER)/include" \ - -I "$(OPENCV_INSTALL_DIR)/include" \ - -I "$(DEEPSTREAM_INSTALL_DIR)/sources/includes" -LIBS := -shared -Wl,-no-undefined \ - -L $(SUBDIR) -lyoloplugin \ - -L$(OPENCV_INSTALL_DIR)/lib -lopencv_core -lopencv_imgproc -lopencv_highgui -lopencv_dnn -lopencv_imgcodecs -Wl,-rpath="$(OPENCV_INSTALL_DIR)/lib"\ - -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart -lnppc -lnppig -lnpps -Wl,-rpath="/usr/local/cuda-$(CUDA_VER)/lib64"\ - -L/usr/local/deepstream/ -lgstnvquery -lgstnvdsmeta -Wl,-rpath,/usr/local/deepstream/ \ - -L $(TENSORRT_INSTALL_DIR)/lib -lnvinfer -lnvinfer_plugin -Wl,-rpath="$(TENSORRT_INSTALL_DIR)/lib" - -OBJS:= $(SRCS:.cpp=.o) - -PKGS:= gstreamer-1.0 gstreamer-base-1.0 gstreamer-video-1.0 -CXXFLAGS+=$(shell pkg-config --cflags $(PKGS)) -LIBS+=$(shell pkg-config --libs $(PKGS)) - -.PHONY: all install clean - -all: $(LIB) - -%.o: %.cpp Makefile - $(CXX) -c -o $@ $(INCS) $(CXXFLAGS) $< - -$(LIB): $(OBJS) $(DEP) Makefile - $(CXX) -o $@ $(OBJS) $(LIBS) $(CXXFLAGS) - -$(DEP): $(DEP_FILES) - @echo "Building yoloplugin_lib" - $(MAKE) -C $(SUBDIR) CXX="$(CXX)" CUDA_VER="$(CUDA_VER)" TENSORRT_INSTALL_DIR="$(TENSORRT_INSTALL_DIR)" OPENCV_INSTALL_DIR="$(OPENCV_INSTALL_DIR)" - -install: $(LIB) - cp -rv $(LIB) /usr/lib/x86_64-linux-gnu/gstreamer-1.0/ - -clean: - rm -rf $(OBJS) $(LIB) - $(MAKE) -C $(SUBDIR) clean diff --git a/sources/gst-yoloplugin/gstyoloplugin.cpp b/sources/gst-yoloplugin/gstyoloplugin.cpp deleted file mode 100644 index d299720..0000000 --- a/sources/gst-yoloplugin/gstyoloplugin.cpp +++ /dev/null @@ -1,702 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#include "gstyoloplugin.h" -#include -#include -#include -#include -#include -#include -#include -#include -GST_DEBUG_CATEGORY_STATIC(gst_yoloplugin_debug); -#define GST_CAT_DEFAULT gst_yoloplugin_debug - -static GQuark _dsmeta_quark = 0; - -/* Enum to identify properties */ -enum -{ - PROP_0, - PROP_UNIQUE_ID, - PROP_PROCESSING_WIDTH, - PROP_PROCESSING_HEIGHT, - PROP_PROCESS_FULL_FRAME, - PROP_GPU_DEVICE_ID -}; - -/* Default values for properties */ -#define DEFAULT_UNIQUE_ID 15 -#define DEFAULT_PROCESSING_WIDTH 640 -#define DEFAULT_PROCESSING_HEIGHT 480 -#define DEFAULT_PROCESS_FULL_FRAME TRUE -#define DEFAULT_GPU_ID 0 - -/* By default NVIDIA Hardware allocated memory flows through the pipeline. We - * will be processing on this type of memory only. */ -#define GST_CAPS_FEATURE_MEMORY_NVMM "memory:NVMM" -static GstStaticPadTemplate gst_yoloplugin_sink_template = GST_STATIC_PAD_TEMPLATE( - "sink", GST_PAD_SINK, GST_PAD_ALWAYS, - GST_STATIC_CAPS(GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_NVMM, "{ RGBA }"))); - -static GstStaticPadTemplate gst_yoloplugin_src_template = GST_STATIC_PAD_TEMPLATE( - "src", GST_PAD_SRC, GST_PAD_ALWAYS, - GST_STATIC_CAPS(GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_NVMM, "{ RGBA }"))); - -/* Define our element type. Standard GObject/GStreamer boilerplate stuff */ -#define gst_yoloplugin_parent_class parent_class -G_DEFINE_TYPE(GstYoloPlugin, gst_yoloplugin, GST_TYPE_BASE_TRANSFORM); - -static void gst_yoloplugin_set_property(GObject* object, guint prop_id, const GValue* value, - GParamSpec* pspec); -static void gst_yoloplugin_get_property(GObject* object, guint prop_id, GValue* value, - GParamSpec* pspec); - -static gboolean gst_yoloplugin_set_caps(GstBaseTransform* btrans, GstCaps* incaps, - GstCaps* outcaps); -static gboolean gst_yoloplugin_start(GstBaseTransform* btrans); -static gboolean gst_yoloplugin_stop(GstBaseTransform* btrans); - -static GstFlowReturn gst_yoloplugin_transform_ip(GstBaseTransform* btrans, GstBuffer* inbuf); - -static void attach_metadata_full_frame(GstYoloPlugin* yoloplugin, GstBuffer* inbuf, - gdouble scale_ratio, YoloPluginOutput* output, - guint batch_id); -static void attach_metadata_object(GstYoloPlugin* yoloplugin, NvDsObjectParams* obj_param, - YoloPluginOutput* output); - -/* Install properties, set sink and src pad capabilities, override the required - * functions of the base class, These are common to all instances of the - * element. - */ -static void gst_yoloplugin_class_init(GstYoloPluginClass* klass) -{ - GObjectClass* gobject_class; - GstElementClass* gstelement_class; - GstBaseTransformClass* gstbasetransform_class; - - gobject_class = (GObjectClass*) klass; - gstelement_class = (GstElementClass*) klass; - gstbasetransform_class = (GstBaseTransformClass*) klass; - - /* Overide base class functions */ - gobject_class->set_property = GST_DEBUG_FUNCPTR(gst_yoloplugin_set_property); - gobject_class->get_property = GST_DEBUG_FUNCPTR(gst_yoloplugin_get_property); - - gstbasetransform_class->set_caps = GST_DEBUG_FUNCPTR(gst_yoloplugin_set_caps); - gstbasetransform_class->start = GST_DEBUG_FUNCPTR(gst_yoloplugin_start); - gstbasetransform_class->stop = GST_DEBUG_FUNCPTR(gst_yoloplugin_stop); - - gstbasetransform_class->transform_ip = GST_DEBUG_FUNCPTR(gst_yoloplugin_transform_ip); - - /* Install properties */ - g_object_class_install_property( - gobject_class, PROP_UNIQUE_ID, - g_param_spec_uint("unique-id", "Unique ID", - "Unique ID for the element. Can be used to identify output of the" - " element", - 0, G_MAXUINT, DEFAULT_UNIQUE_ID, - (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); - - g_object_class_install_property( - gobject_class, PROP_PROCESSING_WIDTH, - g_param_spec_int("processing-width", "Processing Width", - "Width of the input buffer to algorithm", 1, G_MAXINT, - DEFAULT_PROCESSING_WIDTH, - (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); - - g_object_class_install_property( - gobject_class, PROP_PROCESSING_HEIGHT, - g_param_spec_int("processing-height", "Processing Height", - "Height of the input buffer to algorithm", 1, G_MAXINT, - DEFAULT_PROCESSING_HEIGHT, - (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); - - g_object_class_install_property( - gobject_class, PROP_PROCESS_FULL_FRAME, - g_param_spec_boolean("full-frame", "Full frame", - "Enable to process full frame or disable to process objects detected" - "by primary detector", - DEFAULT_PROCESS_FULL_FRAME, - (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); - - g_object_class_install_property( - gobject_class, PROP_GPU_DEVICE_ID, - g_param_spec_uint( - "gpu-id", "Set GPU Device ID", "Set GPU Device ID", 0, G_MAXUINT, 0, - GParamFlags(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY))); - /* Set sink and src pad capabilities */ - gst_element_class_add_pad_template(gstelement_class, - gst_static_pad_template_get(&gst_yoloplugin_src_template)); - gst_element_class_add_pad_template(gstelement_class, - gst_static_pad_template_get(&gst_yoloplugin_sink_template)); - - /* Set metadata describing the element */ - gst_element_class_set_details_simple( - gstelement_class, "NvYolo", "NvYolo", - "Process a 3rdparty example algorithm on objects / full frame", "Nvidia"); -} - -static void gst_yoloplugin_init(GstYoloPlugin* yoloplugin) -{ - GstBaseTransform* btrans = GST_BASE_TRANSFORM(yoloplugin); - - /* We will not be generating a new buffer. Just adding / updating - * metadata. */ - gst_base_transform_set_in_place(GST_BASE_TRANSFORM(btrans), TRUE); - /* We do not want to change the input caps. Set to passthrough. transform_ip - * is still called. */ - gst_base_transform_set_passthrough(GST_BASE_TRANSFORM(btrans), TRUE); - - /* Initialize all property variables to default values */ - yoloplugin->unique_id = DEFAULT_UNIQUE_ID; - yoloplugin->processing_width = DEFAULT_PROCESSING_WIDTH; - yoloplugin->processing_height = DEFAULT_PROCESSING_HEIGHT; - yoloplugin->process_full_frame = DEFAULT_PROCESS_FULL_FRAME; - yoloplugin->gpu_id = DEFAULT_GPU_ID; - /* This quark is required to identify NvDsMeta when iterating through - * the buffer metadatas */ - if (!_dsmeta_quark) _dsmeta_quark = g_quark_from_static_string(NVDS_META_STRING); -} - -/* Function called when a property of the element is set. Standard boilerplate. - */ -static void gst_yoloplugin_set_property(GObject* object, guint prop_id, const GValue* value, - GParamSpec* pspec) -{ - GstYoloPlugin* yoloplugin = GST_YOLOPLUGIN(object); - switch (prop_id) - { - case PROP_UNIQUE_ID: yoloplugin->unique_id = g_value_get_uint(value); break; - case PROP_PROCESSING_WIDTH: yoloplugin->processing_width = g_value_get_int(value); break; - case PROP_PROCESSING_HEIGHT: yoloplugin->processing_height = g_value_get_int(value); break; - case PROP_PROCESS_FULL_FRAME: - yoloplugin->process_full_frame = g_value_get_boolean(value); - break; - case PROP_GPU_DEVICE_ID: yoloplugin->gpu_id = g_value_get_uint(value); break; - default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; - } -} - -/* Function called when a property of the element is requested. Standard - * boilerplate. - */ -static void gst_yoloplugin_get_property(GObject* object, guint prop_id, GValue* value, - GParamSpec* pspec) -{ - GstYoloPlugin* yoloplugin = GST_YOLOPLUGIN(object); - - switch (prop_id) - { - case PROP_UNIQUE_ID: g_value_set_uint(value, yoloplugin->unique_id); break; - case PROP_PROCESSING_WIDTH: g_value_set_int(value, yoloplugin->processing_width); break; - case PROP_PROCESSING_HEIGHT: g_value_set_int(value, yoloplugin->processing_height); break; - case PROP_PROCESS_FULL_FRAME: g_value_set_boolean(value, yoloplugin->process_full_frame); break; - case PROP_GPU_DEVICE_ID: g_value_set_uint(value, yoloplugin->gpu_id); break; - default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; - } -} - -/** - * Initialize all resources and start the output thread - */ -static gboolean gst_yoloplugin_start(GstBaseTransform* btrans) -{ - GstYoloPlugin* yoloplugin = GST_YOLOPLUGIN(btrans); - YoloPluginInitParams init_params = {yoloplugin->processing_width, yoloplugin->processing_height, - yoloplugin->process_full_frame}; - - GstQuery* queryparams = NULL; - guint batch_size = 1; - cudaError_t CUerr = cudaSuccess; - - yoloplugin->batch_size = 1; - queryparams = gst_nvquery_batch_size_new(); - if (gst_pad_peer_query(GST_BASE_TRANSFORM_SINK_PAD(btrans), queryparams) - || gst_pad_peer_query(GST_BASE_TRANSFORM_SRC_PAD(btrans), queryparams)) - { - if (gst_nvquery_batch_size_parse(queryparams, &batch_size)) - { - yoloplugin->batch_size = batch_size; - } - } - GST_DEBUG_OBJECT(yoloplugin, "Setting batch-size %d \n", yoloplugin->batch_size); - gst_query_unref(queryparams); - - /* Algorithm specific initializations and resource allocation. */ - yoloplugin->yolopluginlib_ctx = YoloPluginCtxInit(&init_params, yoloplugin->batch_size); - - GST_DEBUG_OBJECT(yoloplugin, "ctx lib %p \n", yoloplugin->yolopluginlib_ctx); - CUerr = cudaSetDevice(yoloplugin->gpu_id); - if (CUerr != cudaSuccess) - { - g_print("\n *** Unable to set device in %s Line %d\n", __func__, __LINE__); - goto error; - } - - cudaStreamCreate(&yoloplugin->npp_stream); - - // Create host memory for conversion/scaling - CUerr = cudaMallocHost(&yoloplugin->hconv_buf, - yoloplugin->processing_width * yoloplugin->processing_height * 4); - if (CUerr != cudaSuccess) - { - goto error; - } - GST_DEBUG_OBJECT(yoloplugin, "allocated cuda buffer %p \n", yoloplugin->hconv_buf); - - yoloplugin->cvmats = std::vector(yoloplugin->batch_size, nullptr); - for (uint k = 0; k < batch_size; ++k) - { - yoloplugin->cvmats.at(k) = new cv::Mat( - cv::Size(yoloplugin->processing_width, yoloplugin->processing_height), CV_8UC3); - if (!yoloplugin->cvmats.at(k)) goto error; - } - GST_DEBUG_OBJECT(yoloplugin, "created CV Mat\n"); - return TRUE; -error: - if (yoloplugin->hconv_buf) - { - cudaFreeHost(yoloplugin->hconv_buf); - yoloplugin->hconv_buf = NULL; - } - - if (yoloplugin->npp_stream) - { - cudaStreamDestroy(yoloplugin->npp_stream); - yoloplugin->npp_stream = NULL; - } - if (yoloplugin->yolopluginlib_ctx) YoloPluginCtxDeinit(yoloplugin->yolopluginlib_ctx); - return FALSE; -} - -/** - * Stop the output thread and free up all the resources - */ -static gboolean gst_yoloplugin_stop(GstBaseTransform* btrans) -{ - GstYoloPlugin* yoloplugin = GST_YOLOPLUGIN(btrans); - if (yoloplugin->hconv_buf) - { - cudaFreeHost(yoloplugin->hconv_buf); - yoloplugin->hconv_buf = NULL; - GST_DEBUG_OBJECT(yoloplugin, "Freed cuda host buffer \n"); - } - if (yoloplugin->npp_stream) - { - cudaStreamDestroy(yoloplugin->npp_stream); - } - - for (uint i = 0; i < yoloplugin->batch_size; ++i) - { - delete yoloplugin->cvmats.at(i); - } - GST_DEBUG_OBJECT(yoloplugin, "deleted CV Mat \n"); - // Deinit the algorithm library - YoloPluginCtxDeinit(yoloplugin->yolopluginlib_ctx); - GST_DEBUG_OBJECT(yoloplugin, "ctx lib released \n"); - return TRUE; -} - -/** - * Called when source / sink pad capabilities have been negotiated. - */ -static gboolean gst_yoloplugin_set_caps(GstBaseTransform* btrans, GstCaps* incaps, GstCaps* outcaps) -{ - GstYoloPlugin* yoloplugin = GST_YOLOPLUGIN(btrans); - - /* Save the input video information, since this will be required later. */ - gst_video_info_from_caps(&yoloplugin->video_info, incaps); - - return TRUE; -} - -/** - * Scale the entire frame to the processing resolution maintaining aspect ratio. - * Or crop and scale objects to the processing resolution maintaining the aspect - * ratio. Remove the padding requried by hardware and convert from RGBA to RGB - * using openCV. These steps can be skipped if the algorithm can work with - * padded data and/or can work with RGBA. - */ -static GstFlowReturn get_converted_mat_dgpu(GstYoloPlugin* yoloplugin, void* input_buf, - NvOSD_RectParams* crop_rect_params, cv::Mat& out_mat, - gdouble& ratio, gint input_width, gint input_height) -{ - GstFlowReturn flow_ret = GST_FLOW_OK; - cv::Mat in_mat; - gint src_left = (crop_rect_params->left); - gint src_top = (crop_rect_params->top); - gint src_width = (crop_rect_params->width); - gint src_height = (crop_rect_params->height); - NppStatus nppError; - - // size of source - NppiSize oSrcSize = {input_width, input_height}; - - // source ROI - NppiRect oSrcROI = {(gint) 0, (gint) 0, (gint) src_width, (gint) src_height}; - - // Destination ROI - NppiRect DstROI - = {0, 0, (gint) yoloplugin->processing_width, (gint) yoloplugin->processing_height}; - - GST_DEBUG_OBJECT(yoloplugin, "Scaling and converting input buffer\n"); - - // Calculate scaling ratio while maintaining aspect ratio - ratio = MIN(1.0 * yoloplugin->processing_width / crop_rect_params->width, - 1.0 * yoloplugin->processing_height / crop_rect_params->height); - - if ((crop_rect_params->width == 0) || (crop_rect_params->height == 0)) - { - flow_ret = GST_FLOW_ERROR; - goto done; - } - // Memset the memory - cudaMemset(yoloplugin->hconv_buf, 0, - yoloplugin->processing_width * 4 * yoloplugin->processing_height); - - nppSetStream(yoloplugin->npp_stream); - - // Perform cropping and resizing - nppError = nppiResizeSqrPixel_8u_C4R( - (const Npp8u*) input_buf + (src_left + src_top * input_width) * 4, oSrcSize, - input_width * 4, oSrcROI, (Npp8u*) yoloplugin->hconv_buf, yoloplugin->processing_width * 4, - DstROI, ratio, ratio, 0, 0, NPPI_INTER_LINEAR); - if (nppError != NPP_SUCCESS) - { - flow_ret = GST_FLOW_ERROR; - goto done; - } - cudaStreamSynchronize(yoloplugin->npp_stream); - - // Use openCV to remove padding and convert RGBA to RGB. Can be skipped if - // algorithm can handle padded RGBA data. - in_mat = cv::Mat(yoloplugin->processing_height, yoloplugin->processing_width, CV_8UC4, - yoloplugin->hconv_buf, yoloplugin->processing_width * 4); - cv::cvtColor(in_mat, out_mat, CV_RGBA2BGR); -done: - return flow_ret; -} - -/** - * Called when element recieves an input buffer from upstream element. - */ -static GstFlowReturn gst_yoloplugin_transform_ip(GstBaseTransform* btrans, GstBuffer* inbuf) -{ - GstYoloPlugin* yoloplugin = GST_YOLOPLUGIN(btrans); - GstMapInfo in_map_info; - GstFlowReturn flow_ret = GST_FLOW_OK; - gdouble scale_ratio; - std::vector outputs(yoloplugin->batch_size, nullptr); - - cudaError_t CUerr = cudaSuccess; - NvBufSurface* surface = NULL; - guint batch_size = yoloplugin->batch_size; - GstNvStreamMeta* streamMeta = NULL; - - cv::Mat in_mat; - - yoloplugin->frame_num++; - CUerr = cudaSetDevice(yoloplugin->gpu_id); - if (CUerr != cudaSuccess) - { - g_print("\n *** Unable to set device in %s Line %d\n", __func__, __LINE__); - goto done; - } - - memset(&in_map_info, 0, sizeof(in_map_info)); - if (!gst_buffer_map(inbuf, &in_map_info, GST_MAP_READ)) - { - flow_ret = GST_FLOW_ERROR; - goto done; - } - - surface = *((NvBufSurface**) in_map_info.data); - GST_DEBUG_OBJECT(yoloplugin, "Processing Frame %" G_GUINT64_FORMAT " Surface %p\n", - yoloplugin->frame_num, surface); - - /* Stream meta for batched mode */ - streamMeta = gst_buffer_get_nvstream_meta(inbuf); - if (streamMeta && streamMeta->num_filled < yoloplugin->batch_size) - { - batch_size = streamMeta->num_filled; - } - if (yoloplugin->process_full_frame) - { - for (guint i = 0; i < batch_size; i++) - { - NvOSD_RectParams rect_params; - - // Scale the entire frame to processing resolution - rect_params.left = 0; - rect_params.top = 0; - rect_params.width = yoloplugin->video_info.width; - rect_params.height = yoloplugin->video_info.height; - - // Scale and convert the frame - if (get_converted_mat_dgpu(yoloplugin, surface->buf_data[i], &rect_params, - *yoloplugin->cvmats.at(i), scale_ratio, - yoloplugin->video_info.width, yoloplugin->video_info.height) - != GST_FLOW_OK) - { - flow_ret = GST_FLOW_ERROR; - goto done; - } - } - // Process to get the outputs - outputs = YoloPluginProcess(yoloplugin->yolopluginlib_ctx, yoloplugin->cvmats); - - for (uint k = 0; k < outputs.size(); ++k) - { - if (!outputs.at(k)) continue; - // Attach the metadata for the full frame - attach_metadata_full_frame(yoloplugin, inbuf, scale_ratio, outputs.at(k), k); - free(outputs.at(k)); - } - } - else - { - // Using object crops as input to the algorithm. The objects are detected by - // the primary detector - GstMeta* gst_meta; - NvDsMeta* dsmeta; - // NOTE: Initializing state to NULL is essential - gpointer state = NULL; - NvDsFrameMeta* bbparams; - - // Standard way of iterating through buffer metadata - while ((gst_meta = gst_buffer_iterate_meta(inbuf, &state)) != NULL) - { - // Check if this metadata is of NvDsMeta type - if (!gst_meta_api_type_has_tag(gst_meta->info->api, _dsmeta_quark)) continue; - - dsmeta = (NvDsMeta*) gst_meta; - // Check if the metadata of NvDsMeta contains object bounding boxes - if (dsmeta->meta_type != NVDS_META_FRAME_INFO) continue; - - bbparams = (NvDsFrameMeta*) dsmeta->meta_data; - // Check if these parameters have been set by the primary detector / - // tracker - if (bbparams->gie_type != 1) - { - continue; - } - // Iterate through all the objects - for (guint i = 0; i < bbparams->num_rects; i++) - { - NvDsObjectParams* obj_param = &bbparams->obj_params[i]; - - // Crop and scale the object - if (get_converted_mat_dgpu(yoloplugin, surface->buf_data[bbparams->batch_id], - &obj_param->rect_params, *yoloplugin->cvmats.at(i), - scale_ratio, yoloplugin->video_info.width, - yoloplugin->video_info.height) - != GST_FLOW_OK) - { - continue; - } - if (!obj_param->text_params.display_text) - { - bbparams->num_strings++; - } - } - // Process the object crop to obtain label - outputs = YoloPluginProcess(yoloplugin->yolopluginlib_ctx, yoloplugin->cvmats); - - for (uint k = 0; k < outputs.size(); ++k) - { - if (!outputs.at(k)) continue; - NvDsObjectParams* obj_param = &bbparams->obj_params[k]; - // Attach labels for the object - attach_metadata_object(yoloplugin, obj_param, outputs.at(k)); - free(outputs.at(k)); - } - } - } - -done: - gst_buffer_unmap(inbuf, &in_map_info); - return flow_ret; -} - -/** - * Free the metadata allocated in attach_metadata_full_frame - */ -static void free_ds_meta(gpointer meta_data) -{ - NvDsFrameMeta* params = (NvDsFrameMeta*) meta_data; - for (guint i = 0; i < params->num_rects; i++) - { - g_free(params->obj_params[i].text_params.display_text); - } - g_free(params->obj_params); - g_free(params); -} - -/** - * Attach metadata for the full frame. We will be adding a new metadata. - */ -static void attach_metadata_full_frame(GstYoloPlugin* yoloplugin, GstBuffer* inbuf, - gdouble scale_ratio, YoloPluginOutput* output, - guint batch_id) -{ - NvDsMeta* dsmeta; - NvDsFrameMeta* bbparams = (NvDsFrameMeta*) g_malloc0(sizeof(NvDsFrameMeta)); - // Allocate an array of size equal to the number of objects detected - bbparams->obj_params - = (NvDsObjectParams*) g_malloc0(sizeof(NvDsObjectParams) * output->numObjects); - // Should be set to 3 for custom elements - bbparams->gie_type = 3; - // Use HW for overlaying boxes - bbparams->nvosd_mode = NV_OSD_MODE_GPU; - bbparams->batch_id = batch_id; - // Font to be used for label text - static gchar font_name[] = "Arial"; - GST_DEBUG_OBJECT(yoloplugin, "Attaching metadata %d\n", output->numObjects); - for (gint i = 0; i < output->numObjects; i++) - { - YoloPluginObject* obj = &output->object[i]; - NvDsObjectParams* obj_param = &bbparams->obj_params[i]; - NvOSD_RectParams& rect_params = obj_param->rect_params; - NvOSD_TextParams& text_params = obj_param->text_params; - - // Assign bounding box coordinates - rect_params.left = obj->left; - rect_params.top = obj->top; - rect_params.width = obj->width; - rect_params.height = obj->height; - - // Semi-transparent yellow background - rect_params.has_bg_color = 0; - rect_params.bg_color = (NvOSD_ColorParams){1, 1, 0, 0.4}; - // Red border of width 6 - rect_params.border_width = 1; - rect_params.border_color = (NvOSD_ColorParams){1, 0, 0, 1}; - - // Scale the bounding boxes proportionally based on how the object/frame was - // scaled during input - rect_params.left /= scale_ratio; - rect_params.top /= scale_ratio; - rect_params.width /= scale_ratio; - rect_params.height /= scale_ratio; - GST_DEBUG_OBJECT(yoloplugin, - "Attaching rect%d of batch%u" - " left->%u top->%u width->%u" - " height->%u label->%s\n", - i, batch_id, rect_params.left, rect_params.top, rect_params.width, - rect_params.height, obj->label); - bbparams->num_rects++; - - // has_new_info should be set to TRUE whenever adding new/updating - // information to NvDsAttrInfo - obj_param->has_new_info = TRUE; - // Update the approriate element of the attr_info array. Application knows - // that output of this element is available at index "unique_id". - strcpy(obj_param->attr_info[yoloplugin->unique_id].attr_label, obj->label); - // is_attr_label should be set to TRUE indicating that above attr_label field is - // valid - obj_param->attr_info[yoloplugin->unique_id].is_attr_label = 1; - // Obj not yet tracked - obj_param->tracking_id = -1; - - // display_text required heap allocated memory - text_params.display_text = g_strdup(obj->label); - // Display text above the left top corner of the object - text_params.x_offset = rect_params.left; - text_params.y_offset = rect_params.top - 10; - // Set black background for the text - text_params.set_bg_clr = 1; - text_params.text_bg_clr = (NvOSD_ColorParams){0, 0, 0, 1}; - // Font face, size and color - text_params.font_params.font_name = font_name; - text_params.font_params.font_size = 11; - text_params.font_params.font_color = (NvOSD_ColorParams){1, 1, 1, 1}; - bbparams->num_strings++; - } - - // Attach the NvDsFrameMeta structure as NvDsMeta to the buffer. Pass the - // function to be called when freeing the meta_data - dsmeta = gst_buffer_add_nvds_meta(inbuf, bbparams, free_ds_meta); - dsmeta->meta_type = NVDS_META_FRAME_INFO; -} - -/** - * Only update string label in an existing object metadata. No bounding boxes. - * We assume only one label per object is generated - */ -static void attach_metadata_object(GstYoloPlugin* yoloplugin, NvDsObjectParams* obj_param, - YoloPluginOutput* output) -{ - if (output->numObjects == 0) return; - NvOSD_TextParams& text_params = obj_param->text_params; - NvOSD_RectParams& rect_params = obj_param->rect_params; - - // has_new_info should be set to TRUE whenever adding new/updating - // information to NvDsAttrInfo - obj_param->has_new_info = TRUE; - // Update the approriate element of the attr_info array. Application knows - // that output of this element is available at index "unique_id". - strcpy(obj_param->attr_info[yoloplugin->unique_id].attr_label, output->object[0].label); - // is_attr_label should be set to TRUE indicating that above attr_label field is - // valid - obj_param->attr_info[yoloplugin->unique_id].is_attr_label = 1; - // Set black background for the text - // display_text required heap allocated memory - if (text_params.display_text) - { - gchar* conc_string - = g_strconcat(text_params.display_text, " ", output->object[0].label, NULL); - g_free(text_params.display_text); - text_params.display_text = conc_string; - } - else - { - // Display text above the left top corner of the object - text_params.x_offset = rect_params.left; - text_params.y_offset = rect_params.top - 10; - text_params.display_text = g_strdup(output->object[0].label); - // Font face, size and color - text_params.font_params.font_name = "Arial"; - text_params.font_params.font_size = 11; - text_params.font_params.font_color = (NvOSD_ColorParams){1, 1, 1, 1}; - // Set black background for the text - text_params.set_bg_clr = 1; - text_params.text_bg_clr = (NvOSD_ColorParams){0, 0, 0, 1}; - } -} - -/** - * Boiler plate for registering a plugin and an element. - */ -static gboolean yoloplugin_plugin_init(GstPlugin* plugin) -{ - GST_DEBUG_CATEGORY_INIT(gst_yoloplugin_debug, "yolo", 0, "yolo plugin"); - - return gst_element_register(plugin, "nvyolo", GST_RANK_PRIMARY, GST_TYPE_YOLOPLUGIN); -} - -GST_PLUGIN_DEFINE(GST_VERSION_MAJOR, GST_VERSION_MINOR, yoloplugin, DESCRIPTION, - yoloplugin_plugin_init, VERSION, LICENSE, BINARY_PACKAGE, URL) diff --git a/sources/gst-yoloplugin/gstyoloplugin.h b/sources/gst-yoloplugin/gstyoloplugin.h deleted file mode 100644 index dbf3246..0000000 --- a/sources/gst-yoloplugin/gstyoloplugin.h +++ /dev/null @@ -1,117 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#ifndef __GST_YOLOPLUGIN_H__ -#define __GST_YOLOPLUGIN_H__ - -#include -#include - -/* Open CV headers */ -#include "opencv2/highgui/highgui.hpp" -#include "opencv2/imgproc/imgproc.hpp" - -#include "gst-nvquery.h" -#include "gstnvdsmeta.h" -#include "gstnvstreammeta.h" -#include "nvbuffer.h" -#include "yoloplugin_lib/yoloplugin_lib.h" -#include -#include - -/* Package and library details required for plugin_init */ -#define PACKAGE "nvyolo" -#define VERSION "1.0" -#define LICENSE "Proprietary" -#define DESCRIPTION "NVIDIA example plugin for integration with DeepStream on DGPU" -#define BINARY_PACKAGE "NVIDIA DeepStream 3rdparty IP integration example plugin" -#define URL "http://nvidia.com/" - -G_BEGIN_DECLS -/* Standard boilerplate stuff */ -typedef struct _GstYoloPlugin GstYoloPlugin; -typedef struct _GstYoloPluginClass GstYoloPluginClass; - -/* Standard boilerplate stuff */ -#define GST_TYPE_YOLOPLUGIN (gst_yoloplugin_get_type()) -#define GST_YOLOPLUGIN(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_YOLOPLUGIN, GstYoloPlugin)) -#define GST_YOLOPLUGIN_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_YOLOPLUGIN, GstYoloPluginClass)) -#define GST_YOLOPLUGIN_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), GST_TYPE_YOLOPLUGIN, GstYoloPluginClass)) -#define GST_IS_YOLOPLUGIN(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_YOLOPLUGIN)) -#define GST_IS_YOLOPLUGIN_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_YOLOPLUGIN)) -#define GST_YOLOPLUGIN_CAST(obj) ((GstYoloPlugin*) (obj)) - -struct _GstYoloPlugin -{ - GstBaseTransform base_trans; - - // Context of the custom algorithm library - YoloPluginCtx* yolopluginlib_ctx; - - // Unique ID of the element. The labels generated by the element will be - // updated at index `unique_id` of attr_info array in NvDsObjectParams. - guint unique_id; - - // Frame number of the current input buffer - guint64 frame_num; - - // NPP Stream used for allocating the CUDA task - cudaStream_t npp_stream; - - // the scratch conversion host buffer for DGPU - void* hconv_buf; - - // OpenCV mat to remove padding and convert RGBA to RGB - std::vector cvmats; - - // Input video info (resolution, color format, framerate, etc) - GstVideoInfo video_info; - - // Resolution at which frames/objects should be processed - gint processing_width; - gint processing_height; - - // Amount of objects processed in single call to algorithm - guint batch_size; - - // GPU ID on which we expect to execute the task - guint gpu_id; - - // Boolean indicating if entire frame or cropped objects should be processed - gboolean process_full_frame; -}; - -// Boiler plate stuff -struct _GstYoloPluginClass -{ - GstBaseTransformClass parent_class; -}; - -GType gst_yoloplugin_get_type(void); - -G_END_DECLS -#endif /* __GST_YOLOPLUGIN_H__ */ diff --git a/sources/gst-yoloplugin/yoloplugin_lib/Makefile b/sources/gst-yoloplugin/yoloplugin_lib/Makefile deleted file mode 100644 index bc09c98..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/Makefile +++ /dev/null @@ -1,56 +0,0 @@ -# MIT License - -# Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -SRCS:= $(wildcard *.cpp) -BUILD_PATH:= ./build/ -OBJS= $(patsubst %, $(BUILD_PATH)%, $(SRCS:.cpp=.o)) -DEPS:= $(SRCS) -DEPS+= $(wildcard *.h) -TARGET:= libyoloplugin.a -INCS:= -I"$(TENSORRT_INSTALL_DIR)/include" \ - -I"/usr/local/cuda-$(CUDA_VER)/include" \ - -I "$(OPENCV_INSTALL_DIR)/include" -LIBS:= -L"$(TENSORRT_INSTALL_DIR)/lib" -lnvinfer -lnvinfer_plugin -Wl,-rpath="$(TENSORRT_INSTALL_DIR)/lib" \ - -L"/usr/local/cuda-$(CUDA_VER)/lib64" -lcudart -lcublas -lcurand -Wl,-rpath="/usr/local/cuda-$(CUDA_VER)/lib64" \ - -L "$(OPENCV_INSTALL_DIR)/lib" -lopencv_core -lopencv_imgproc -lopencv_imgcodecs -lopencv_highgui -lopencv_dnn -Wl,-rpath="$(OPENCV_INSTALL_DIR)/lib" -CXXFLAGS:= -O2 -std=c++11 -lstdc++fs -fPIC -Wall -Wunused-function -Wunused-variable `pkg-config --cflags glib-2.0` - -.PHONY: all dirs clean deps - -all: dirs deps - ar rcs $(TARGET) $(OBJS) - -dirs: - if [ ! -d "models" ]; then mkdir -p models; fi - if [ ! -d "calibration" ]; then mkdir -p calibration; fi - if [ ! -d "build" ]; then mkdir -p build; fi - if [ ! -d "detections" ]; then mkdir -p detections; fi - -deps: $(DEPS) $(OBJS) - -$(BUILD_PATH)%.o: %.cpp %.h - $(CXX) $(INCS) -c -o $@ $(CXXFLAGS) $< - -clean: - rm -f ./build/* - rm -f ./*.a \ No newline at end of file diff --git a/sources/gst-yoloplugin/yoloplugin_lib/calibration/yolov2-calibration.table b/sources/gst-yoloplugin/yoloplugin_lib/calibration/yolov2-calibration.table deleted file mode 100644 index 712d70e..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/calibration/yolov2-calibration.table +++ /dev/null @@ -1,56 +0,0 @@ -1 -(Unnamed ITensor* 46): 3da24548 -(Unnamed ITensor* 52): 3dc28553 -(Unnamed ITensor* 44): 3d5e175c -(Unnamed ITensor* 43): 3d339686 -(Unnamed ITensor* 42): 3dd1214d -(Unnamed ITensor* 64): 3dc36ed6 -(Unnamed ITensor* 47): 3d1b5583 -(Unnamed ITensor* 40): 3d81127d -(Unnamed ITensor* 56): 3d2fd5f8 -(Unnamed ITensor* 21): 3d622768 -(Unnamed ITensor* 37): 3d33781d -(Unnamed ITensor* 16): 3e2152dd -(Unnamed ITensor* 69) copy: 3d222fbf -(Unnamed ITensor* 18): 3d610706 -(Unnamed ITensor* 23): 3debb209 -(Unnamed ITensor* 55): 3e702210 -(Unnamed ITensor* 33): 3e09ccb1 -(Unnamed ITensor* 30): 3dc55e3c -(Unnamed ITensor* 7): 3d171917 -(Unnamed ITensor* 24): 3d96faef -(Unnamed ITensor* 27): 3d5e2f66 -(Unnamed ITensor* 58): 3f0779df -(Unnamed ITensor* 11): 3d939856 -(Unnamed ITensor* 10): 3e05913e -(Unnamed ITensor* 53): 3d2a0b6f -(Unnamed ITensor* 62): 3d1adfe4 -(Unnamed ITensor* 3): 3d0c1fd6 -(Unnamed ITensor* 68): 3d222fbf -(Unnamed ITensor* 39): 3e087a5f -(Unnamed ITensor* 2): 3dba1b70 -(Unnamed ITensor* 49): 3e0544a4 -(Unnamed ITensor* 20): 3dfd0594 -(Unnamed ITensor* 65): 3d1a410f -(Unnamed ITensor* 13): 3ddd5a9c -(Unnamed ITensor* 50): 3d5d9887 -(Unnamed ITensor* 67): 3dba55d2 -(Unnamed ITensor* 34): 3d8a5820 -(Unnamed ITensor* 73): 3d110c5d -(Unnamed ITensor* 74): 3e59164f -region_32: 3be7c677 -(Unnamed ITensor* 28): 3d724d98 -(Unnamed ITensor* 17): 3d454da6 -(Unnamed ITensor* 69): 3d222fbf -(Unnamed ITensor* 36): 3dab29b7 -(Unnamed ITensor* 72): 3db01add -(Unnamed ITensor* 59): 3e2f06e9 -(Unnamed ITensor* 65) copy: 3d1a410f -(Unnamed ITensor* 14): 3d99a147 -(Unnamed ITensor* 4): 3db27290 -(Unnamed ITensor* 8): 3db24193 -data: 3c008912 -(Unnamed ITensor* 26): 3df60e0b -(Unnamed ITensor* 6): 3e0e983f -(Unnamed ITensor* 31): 3d3a323a -(Unnamed ITensor* 61): 3dab505f diff --git a/sources/gst-yoloplugin/yoloplugin_lib/calibration/yolov3-calibration.table b/sources/gst-yoloplugin/yoloplugin_lib/calibration/yolov3-calibration.table deleted file mode 100644 index 1671f0e..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/calibration/yolov3-calibration.table +++ /dev/null @@ -1,182 +0,0 @@ -1 -(Unnamed ITensor* 6): 3d480058 -(Unnamed ITensor* 140): 3c9d735e -(Unnamed ITensor* 94): 3d86d842 -(Unnamed ITensor* 109): 3d0de0c2 -(Unnamed ITensor* 225): 3d69e3af -(Unnamed ITensor* 9): 3d2b14e2 -(Unnamed ITensor* 45): 3d38d115 -(Unnamed ITensor* 144): 3ce2a41e -(Unnamed ITensor* 191): 3d419fad -(Unnamed ITensor* 63): 3d9076ad -(Unnamed ITensor* 19): 3cad3e5a -(Unnamed ITensor* 32): 3d59d53c -(Unnamed ITensor* 49): 3d5a8754 -(Unnamed ITensor* 175): 3ca0158f -(Unnamed ITensor* 38): 3d096390 -(Unnamed ITensor* 56): 3d47a32f -(Unnamed ITensor* 143): 3da5e352 -(Unnamed ITensor* 47): 3d30aca6 -(Unnamed ITensor* 42): 3d86de9e -(Unnamed ITensor* 243): 3de3a99e -(Unnamed ITensor* 43): 3d15d776 -(Unnamed ITensor* 13): 3d95b721 -(Unnamed ITensor* 46): 3cb9f239 -(Unnamed ITensor* 67): 3cf030c7 -(Unnamed ITensor* 50): 3d07a2a5 -(Unnamed ITensor* 116): 3cf6bff0 -(Unnamed ITensor* 52): 3d753c1a -(Unnamed ITensor* 187): 3dcbdb33 -(Unnamed ITensor* 234): 3dbf01c1 -(Unnamed ITensor* 134): 3da99a44 -(Unnamed ITensor* 66): 3d8855fa -(Unnamed ITensor* 218): 3decf16e -(Unnamed ITensor* 137): 3cee3c09 -(Unnamed ITensor* 161): 3cb958e1 -(Unnamed ITensor* 118): 3d857d10 -(Unnamed ITensor* 101): 3d82c343 -(Unnamed ITensor* 244): 3d638607 -(Unnamed ITensor* 60): 3caab833 -(Unnamed ITensor* 210): 3d88d870 -(Unnamed ITensor* 141): 3daa8ddf -(Unnamed ITensor* 150): 3dd58289 -(Unnamed ITensor* 228): 3e384d5f -(Unnamed ITensor* 151): 3d06f62d -(Unnamed ITensor* 158): 3d16ed60 -(Unnamed ITensor* 212): 3de9ae3b -(Unnamed ITensor* 202): 3c010a14 -(Unnamed ITensor* 241): 3d545b20 -(Unnamed ITensor* 213): 3d68ac27 -(Unnamed ITensor* 171): 3cdd5993 -(Unnamed ITensor* 12): 3d6b2858 -(Unnamed ITensor* 203): 3c010a14 -(Unnamed ITensor* 238): 3d864ae7 -(Unnamed ITensor* 201): 3d8d3035 -(Unnamed ITensor* 209): 3de5bc94 -(Unnamed ITensor* 165): 3d331de2 -(Unnamed ITensor* 8): 3d8868e8 -(Unnamed ITensor* 179): 3d8ca062 -(Unnamed ITensor* 174): 3e105192 -(Unnamed ITensor* 230): 3c010a14 -(Unnamed ITensor* 219): 3d1aeedd -yolo_83: 3e664693 -(Unnamed ITensor* 204): 3d8d3035 -(Unnamed ITensor* 221): 3df4eae7 -(Unnamed ITensor* 247): 3d5031dd -(Unnamed ITensor* 249): 3de89792 -(Unnamed ITensor* 252): 3de288e0 -(Unnamed ITensor* 205): 3d8d3035 -(Unnamed ITensor* 246): 3dcfed2f -yolo_107: 3ec82cfa -(Unnamed ITensor* 237): 3dea7fc9 -(Unnamed ITensor* 216): 3d125f60 -(Unnamed ITensor* 29): 3d014dba -(Unnamed ITensor* 177): 3dea77e2 -(Unnamed ITensor* 250): 3d449db5 -(Unnamed ITensor* 181): 3dcace38 -(Unnamed ITensor* 194): 3d213dd9 -(Unnamed ITensor* 240): 3ddd0139 -(Unnamed ITensor* 229): 3dbf01c1 -yolo_95: 3e5de27c -(Unnamed ITensor* 188): 3d2e6377 -(Unnamed ITensor* 215): 3df9d430 -(Unnamed ITensor* 185): 3d633ad7 -(Unnamed ITensor* 184): 3dbce750 -(Unnamed ITensor* 233): 3dbf01c1 -(Unnamed ITensor* 70): 3d651dea -(Unnamed ITensor* 78): 3cf408a2 -(Unnamed ITensor* 102): 3d2baebc -(Unnamed ITensor* 71): 3cf85438 -(Unnamed ITensor* 193): 3dcc2522 -(Unnamed ITensor* 73): 3dcead0b -(Unnamed ITensor* 222): 3d1adff9 -(Unnamed ITensor* 74): 3d25ade7 -(Unnamed ITensor* 97): 3d0dfb0b -(Unnamed ITensor* 87): 3db7fdf1 -(Unnamed ITensor* 106): 3d2f5204 -(Unnamed ITensor* 113): 3d32215e -(Unnamed ITensor* 61): 3d573886 -(Unnamed ITensor* 53): 3cb34165 -(Unnamed ITensor* 82): 3dcf9b37 -(Unnamed ITensor* 84): 3d8a9708 -(Unnamed ITensor* 167): 3ded846c -(Unnamed ITensor* 95): 3d20bbf4 -(Unnamed ITensor* 105): 3cb20ea6 -(Unnamed ITensor* 88): 3d263df4 -(Unnamed ITensor* 170): 3db1ef6d -(Unnamed ITensor* 231): 3c010a14 -(Unnamed ITensor* 75): 3dc20560 -(Unnamed ITensor* 122): 3d8f58bf -(Unnamed ITensor* 81): 3d1ed1eb -(Unnamed ITensor* 129): 3d991825 -(Unnamed ITensor* 164): 3cbc9665 -(Unnamed ITensor* 147): 3d11b4ae -(Unnamed ITensor* 115): 3d9c3ef5 -(Unnamed ITensor* 139): 3db40637 -(Unnamed ITensor* 232): 3dbf01c1 -(Unnamed ITensor* 92): 3d28510f -(Unnamed ITensor* 108): 3d992653 -(Unnamed ITensor* 119): 3cd5d4ee -(Unnamed ITensor* 59): 3d3e5a53 -(Unnamed ITensor* 132): 3dd117e1 -(Unnamed ITensor* 182): 3d594848 -(Unnamed ITensor* 156): 3d863ed1 -(Unnamed ITensor* 112): 3cbe0243 -(Unnamed ITensor* 130): 3cfd8364 -(Unnamed ITensor* 148): 3dc2c6ac -(Unnamed ITensor* 36): 3cf3427c -(Unnamed ITensor* 99): 3cfce3a4 -(Unnamed ITensor* 98): 3c0eee44 -(Unnamed ITensor* 35): 3d548b08 -(Unnamed ITensor* 85): 3d001bbe -(Unnamed ITensor* 15): 3dae828f -(Unnamed ITensor* 40): 3d1b702b -(Unnamed ITensor* 89): 3dbf0a7e -(Unnamed ITensor* 160): 3e017d83 -(Unnamed ITensor* 91): 3d9aa5a7 -(Unnamed ITensor* 111): 3d3e1b6c -(Unnamed ITensor* 33): 3d0f4df8 -(Unnamed ITensor* 127): 3d845514 -(Unnamed ITensor* 28): 3d993195 -data: 3c008912 -(Unnamed ITensor* 26): 3d000e3f -(Unnamed ITensor* 178): 3d1cf86a -(Unnamed ITensor* 23): 3d5a4d48 -(Unnamed ITensor* 18): 3d40a507 -(Unnamed ITensor* 22): 3cca62a8 -(Unnamed ITensor* 157): 3ca3b5e4 -(Unnamed ITensor* 16): 3d6bbe1a -(Unnamed ITensor* 197): 3d3a1432 -(Unnamed ITensor* 206): 3d8d3035 -(Unnamed ITensor* 172): 3d3cba64 -(Unnamed ITensor* 125): 3d7a1ec4 -(Unnamed ITensor* 146): 3de0e60d -(Unnamed ITensor* 68): 3d7ff0c4 -(Unnamed ITensor* 3): 3d5b765b -(Unnamed ITensor* 80): 3dba1247 -(Unnamed ITensor* 253): 3da25a76 -(Unnamed ITensor* 123): 3cefaa58 -(Unnamed ITensor* 21): 3d3e011a -(Unnamed ITensor* 154): 3d2b1ea3 -(Unnamed ITensor* 163): 3d6c3fa9 -(Unnamed ITensor* 133): 3d392ff1 -(Unnamed ITensor* 57): 3d15caaa -(Unnamed ITensor* 30): 3d6b151d -(Unnamed ITensor* 2): 3dabf6cf -(Unnamed ITensor* 136): 3d9f152f -(Unnamed ITensor* 25): 3da51855 -(Unnamed ITensor* 224): 3db7d93c -(Unnamed ITensor* 54): 3d485188 -(Unnamed ITensor* 104): 3d2e7b88 -(Unnamed ITensor* 11): 3e118356 -(Unnamed ITensor* 190): 3dc2cc59 -(Unnamed ITensor* 200): 3e0e9eb9 -(Unnamed ITensor* 5): 3d95eb27 -(Unnamed ITensor* 196): 3d9031c4 -(Unnamed ITensor* 126): 3ccbd4bf -(Unnamed ITensor* 120): 3d4207e1 -(Unnamed ITensor* 77): 3d71dbf2 -(Unnamed ITensor* 39): 3c6cc9f8 -(Unnamed ITensor* 153): 3e0e376c -(Unnamed ITensor* 168): 3ceb5a91 -(Unnamed ITensor* 64): 3d0bd1fc diff --git a/sources/gst-yoloplugin/yoloplugin_lib/calibrator.cpp b/sources/gst-yoloplugin/yoloplugin_lib/calibrator.cpp deleted file mode 100644 index a528370..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/calibrator.cpp +++ /dev/null @@ -1,105 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#include "calibrator.h" -#include -#include -#include - -Int8EntropyCalibrator::Int8EntropyCalibrator(const uint& batchSize, - const std::string& calibrationSetPath, - const std::string& calibTableFilePath, - const uint64_t& inputSize, const uint& inputH, - const uint& inputW, const std::string& inputBlobName) : - m_BatchSize(batchSize), - m_InputH(inputH), - m_InputW(inputW), - m_InputSize(inputSize), - m_InputCount(batchSize * inputSize), - m_InputBlobName(inputBlobName.c_str()), - m_CalibTableFilePath(calibTableFilePath), - m_ImageIndex(0) -{ - m_ImageList = loadImageList(calibrationSetPath); - m_ImageList.resize(static_cast(m_ImageList.size() / m_BatchSize) * m_BatchSize); - std::random_shuffle(m_ImageList.begin(), m_ImageList.end(), [](int i) { return rand() % i; }); - NV_CUDA_CHECK(cudaMalloc(&m_DeviceInput, m_InputCount * sizeof(float))); -} - -bool Int8EntropyCalibrator::getBatch(void* bindings[], const char* names[], int nbBindings) -{ - if (m_ImageIndex + m_BatchSize >= m_ImageList.size()) return false; - - // Load next batch - std::vector dsImages(m_BatchSize); - for (uint j = m_ImageIndex; j < m_ImageIndex + m_BatchSize; ++j) - { - dsImages.at(j - m_ImageIndex) = DsImage(m_ImageList.at(j), m_InputH, m_InputW); - } - m_ImageIndex += m_BatchSize; - - cv::Mat trtInput = blobFromDsImages(dsImages, m_InputH, m_InputW); - - NV_CUDA_CHECK(cudaMemcpy(m_DeviceInput, trtInput.ptr(0), m_InputCount * sizeof(float), - cudaMemcpyHostToDevice)); - assert(!strcmp(names[0], m_InputBlobName)); - bindings[0] = m_DeviceInput; - return true; -} - -const void* Int8EntropyCalibrator::readCalibrationCache(size_t& length) -{ - void* output; - m_CalibrationCache.clear(); - assert(!m_CalibTableFilePath.empty()); - std::ifstream input(m_CalibTableFilePath, std::ios::binary); - input >> std::noskipws; - if (m_ReadCache && input.good()) - std::copy(std::istream_iterator(input), std::istream_iterator(), - std::back_inserter(m_CalibrationCache)); - - length = m_CalibrationCache.size(); - if (length) - { - std::cout << "Using cached calibration table to build the engine" << std::endl; - output = &m_CalibrationCache[0]; - } - - else - { - std::cout << "New calibration table will be created to build the engine" << std::endl; - output = nullptr; - } - - return output; -} - -void Int8EntropyCalibrator::writeCalibrationCache(const void* cache, size_t length) -{ - assert(!m_CalibTableFilePath.empty()); - std::ofstream output(m_CalibTableFilePath, std::ios::binary); - output.write(reinterpret_cast(cache), length); - output.close(); -} \ No newline at end of file diff --git a/sources/gst-yoloplugin/yoloplugin_lib/calibrator.h b/sources/gst-yoloplugin/yoloplugin_lib/calibrator.h deleted file mode 100644 index c4e334c..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/calibrator.h +++ /dev/null @@ -1,60 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ -#ifndef _CALIBRATOR_H_ -#define _CALIBRATOR_H_ - -#include "NvInfer.h" -#include "ds_image.h" -#include "trt_utils.h" - -class Int8EntropyCalibrator : public nvinfer1::IInt8EntropyCalibrator -{ -public: - Int8EntropyCalibrator(const uint& batchSize, const std::string& calibrationSetPath, - const std::string& calibTableFilePath, const uint64_t& inputSize, - const uint& inputH, const uint& inputW, const std::string& inputBlobName); - virtual ~Int8EntropyCalibrator() { NV_CUDA_CHECK(cudaFree(m_DeviceInput)); } - - int getBatchSize() const override { return m_BatchSize; } - bool getBatch(void* bindings[], const char* names[], int nbBindings) override; - const void* readCalibrationCache(size_t& length) override; - void writeCalibrationCache(const void* cache, size_t length) override; - -private: - const uint m_BatchSize; - const uint m_InputH; - const uint m_InputW; - const uint64_t m_InputSize; - const uint64_t m_InputCount; - const char* m_InputBlobName; - const std::string m_CalibTableFilePath{nullptr}; - uint m_ImageIndex; - bool m_ReadCache{true}; - void* m_DeviceInput{nullptr}; - std::vector m_ImageList; - std::vector m_CalibrationCache; -}; - -#endif \ No newline at end of file diff --git a/sources/gst-yoloplugin/yoloplugin_lib/data/calibration_images.txt b/sources/gst-yoloplugin/yoloplugin_lib/data/calibration_images.txt deleted file mode 100644 index 2fc40cf..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/data/calibration_images.txt +++ /dev/null @@ -1,5 +0,0 @@ -/path/to/image1.jpg -/path/to/image2.jpg -. -. -. diff --git a/sources/gst-yoloplugin/yoloplugin_lib/data/test_images.txt b/sources/gst-yoloplugin/yoloplugin_lib/data/test_images.txt deleted file mode 100644 index 2fc40cf..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/data/test_images.txt +++ /dev/null @@ -1,5 +0,0 @@ -/path/to/image1.jpg -/path/to/image2.jpg -. -. -. diff --git a/sources/gst-yoloplugin/yoloplugin_lib/ds_image.cpp b/sources/gst-yoloplugin/yoloplugin_lib/ds_image.cpp deleted file mode 100644 index fa1d82a..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/ds_image.cpp +++ /dev/null @@ -1,125 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ -#include "ds_image.h" -#include - -DsImage::DsImage() : - m_Height(0), - m_Width(0), - m_XOffset(0), - m_YOffset(0), - m_ScalingFactor(0.0), - m_RNG(cv::RNG(unsigned(std::time(0)))), - m_ImageName() -{ -} - -DsImage::DsImage(const std::string& path, const int& inputH, const int& inputW) : - m_Height(0), - m_Width(0), - m_XOffset(0), - m_YOffset(0), - m_ScalingFactor(0.0), - m_RNG(cv::RNG(unsigned(std::time(0)))), - m_ImageName() -{ - m_ImageName = std::experimental::filesystem::path(path).stem().string(); - m_OrigImage = cv::imread(path, CV_LOAD_IMAGE_COLOR); - - if (!m_OrigImage.data || m_OrigImage.cols <= 0 || m_OrigImage.rows <= 0) - { - std::cout << "Unable to open image : " << path << std::endl; - assert(0); - } - - if (m_OrigImage.channels() != 3) - { - std::cout << "Non RGB images are not supported : " << path << std::endl; - assert(0); - } - - m_OrigImage.copyTo(m_MarkedImage); - m_Height = m_OrigImage.rows; - m_Width = m_OrigImage.cols; - - // resize the DsImage with scale - float dim = std::max(m_Height, m_Width); - int resizeH = ((m_Height / dim) * inputH); - int resizeW = ((m_Width / dim) * inputW); - m_ScalingFactor = static_cast(resizeH) / static_cast(m_Height); - - // Additional checks for images with non even dims - if ((inputW - resizeW) % 2) resizeW--; - if ((inputH - resizeH) % 2) resizeH--; - assert((inputW - resizeW) % 2 == 0); - assert((inputH - resizeH) % 2 == 0); - - m_XOffset = (inputW - resizeW) / 2; - m_YOffset = (inputH - resizeH) / 2; - - assert(2 * m_XOffset + resizeW == inputW); - assert(2 * m_YOffset + resizeH == inputH); - - // resizing - cv::resize(m_OrigImage, m_LetterboxImage, cv::Size(resizeW, resizeH), 0, 0, cv::INTER_CUBIC); - // letterboxing - cv::copyMakeBorder(m_LetterboxImage, m_LetterboxImage, m_YOffset, m_YOffset, m_XOffset, - m_XOffset, cv::BORDER_CONSTANT, cv::Scalar(128, 128, 128)); - - m_LetterboxImage.convertTo(m_LetterboxImage, CV_32FC3, 1 / 255.0); - cv::threshold(m_LetterboxImage, m_LetterboxImage, 1.0, 1.0, cv::ThresholdTypes::THRESH_TRUNC); - // converting to RGB - cv::cvtColor(m_LetterboxImage, m_LetterboxImage, CV_BGR2RGB); -} - -void DsImage::addBBox(BBoxInfo box, const std::string& labelName) -{ - m_Bboxes.push_back(box); - const int x = box.box.x1; - const int y = box.box.y1; - const int w = box.box.x2 - box.box.x1; - const int h = box.box.y2 - box.box.y1; - const cv::Scalar color - = cv::Scalar(m_RNG.uniform(0, 255), m_RNG.uniform(0, 255), m_RNG.uniform(0, 255)); - - cv::rectangle(m_MarkedImage, cv::Rect(x, y, w, h), color, 1); - const cv::Size tsize - = cv::getTextSize(labelName, cv::FONT_HERSHEY_COMPLEX_SMALL, 0.5, 1, nullptr); - cv::rectangle(m_MarkedImage, cv::Rect(x, y, tsize.width + 3, tsize.height + 4), color, -1); - cv::putText(m_MarkedImage, labelName.c_str(), cv::Point(x, y + tsize.height), - cv::FONT_HERSHEY_COMPLEX_SMALL, 0.5, cv::Scalar(255, 255, 255), 1, CV_AA); -} - -void DsImage::showImage(const std::string& windowName) const -{ - cv::namedWindow(windowName); - cv::imshow(windowName.c_str(), m_MarkedImage); - cv::waitKey(0); -} - -void DsImage::saveImageJPEG(const std::string& dirPath) const -{ - cv::imwrite(dirPath + m_ImageName + ".jpeg", m_MarkedImage); -} \ No newline at end of file diff --git a/sources/gst-yoloplugin/yoloplugin_lib/ds_image.h b/sources/gst-yoloplugin/yoloplugin_lib/ds_image.h deleted file mode 100644 index 1079bdd..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/ds_image.h +++ /dev/null @@ -1,63 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ -#ifndef __IMAGE_H__ -#define __IMAGE_H__ - -#include "trt_utils.h" - -struct BBoxInfo; - -class DsImage -{ -public: - DsImage(); - DsImage(const std::string& path, const int& inputH, const int& inputW); - int getImageHeight() const { return m_Height; } - int getImageWidth() const { return m_Width; } - cv::Mat getLetterBoxedImage() const { return m_LetterboxImage; } - void addBBox(BBoxInfo box, const std::string& labelName); - void showImage(const std::string& windowName = "Detections") const; - void saveImageJPEG(const std::string& dirPath) const; - -private: - int m_Height; - int m_Width; - int m_XOffset; - int m_YOffset; - float m_ScalingFactor; - std::string m_ImagePath; - cv::RNG m_RNG; - std::string m_ImageName; - std::vector m_Bboxes; - - // unaltered original Image - cv::Mat m_OrigImage; - // letterboxed Image given to the network as input - cv::Mat m_LetterboxImage; - // final image marked with the bounding boxes - cv::Mat m_MarkedImage; -}; - -#endif diff --git a/sources/gst-yoloplugin/yoloplugin_lib/network_config.cpp b/sources/gst-yoloplugin/yoloplugin_lib/network_config.cpp deleted file mode 100644 index c42a4ad..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/network_config.cpp +++ /dev/null @@ -1,123 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#include "network_config.h" - -namespace config -{ - -// Common global vars -const bool kPRINT_PERF_INFO = false; -const bool kPRINT_PRED_INFO = false; -const bool kSAVE_DETECTIONS = false; - -const std::string kPRECISION = "kFLOAT"; -const std::string kINPUT_BLOB_NAME = "data"; -const uint kINPUT_H = 416; -const uint kINPUT_W = 416; -const uint kINPUT_C = 3; -const uint64_t kINPUT_SIZE = kINPUT_C * kINPUT_H * kINPUT_W; -const uint kOUTPUT_CLASSES = 80; -const std::vector kCLASS_NAMES - = {"person", "bicycle", "car", "motorbike", - "aeroplane", "bus", "train", "truck", - "boat", "traffic light", "fire hydrant", "stop sign", - "parking meter", "bench", "bird", "cat", - "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", - "backpack", "umbrella", "handbag", "tie", - "suitcase", "frisbee", "skis", "snowboard", - "sports ball", "kite", "baseball bat", "baseball glove", - "skateboard", "surfboard", "tennis racket", "bottle", - "wine glass", "cup", "fork", "knife", - "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", - "hot dog", "pizza", "donut", "cake", - "chair", "sofa", "pottedplant", "bed", - "diningtable", "toilet", "tvmonitor", "laptop", - "mouse", "remote", "keyboard", "cell phone", - "microwave", "oven", "toaster", "sink", - "refrigerator", "book", "clock", "vase", - "scissors", "teddy bear", "hair drier", "toothbrush"}; -const std::string kDS_LIB_PATH = "sources/gst-yoloplugin/yoloplugin_lib/"; -const std::string kMODELS_PATH = kDS_LIB_PATH + "models/"; -const std::string kDETECTION_RESULTS_PATH = kDS_LIB_PATH + "detections/"; -const std::string kCALIBRATION_SET = kDS_LIB_PATH + "data/calibration_images.txt"; -const std::string kTEST_IMAGES = kDS_LIB_PATH + "data/test_images.txt"; - -// Model V2 specific common global vars -#ifdef MODEL_V2 - -const float kPROB_THRESH = 0.5f; -const float kNMS_THRESH = 0.5f; -const std::string kYOLO_CONFIG_PATH = kDS_LIB_PATH + "data/yolov2.cfg"; -const std::string kTRAINED_WEIGHTS_PATH = kDS_LIB_PATH + "data/yolov2.weights"; -const std::string kNETWORK_TYPE = "yolov2"; -const std::string kCALIB_TABLE_PATH = kDS_LIB_PATH + "calibration/yolov2-calibration.table"; -const uint kBBOXES = 5; -// Anchors have been converted to network input resolution {0.57273, 0.677385, 1.87446, -// 2.06253, 3.33843, 5.47434, 7.88282, 3.52778, 9.77052, 9.16828} x 32 (stride) -const std::vector kANCHORS = {18.32736, 21.67632, 59.98272, 66.00096, 106.82976, - 175.17888, 252.25024, 112.88896, 312.65664, 293.38496}; -#endif - -// Model V2 specific unique global vars -const uint kSTRIDE = 32; -const uint kGRID_SIZE = kINPUT_H / kSTRIDE; -const uint64_t kOUTPUT_SIZE = kGRID_SIZE * kGRID_SIZE * (kBBOXES * (5 + kOUTPUT_CLASSES)); -const std::string kOUTPUT_BLOB_NAME = "region_32"; - -// Model V3 specific common global vars -#ifdef MODEL_V3 - -const float kPROB_THRESH = 0.7f; -const float kNMS_THRESH = 0.5f; -const std::string kYOLO_CONFIG_PATH = kDS_LIB_PATH + "data/yolov3.cfg"; -const std::string kTRAINED_WEIGHTS_PATH = kDS_LIB_PATH + "data/yolov3.weights"; -const std::string kNETWORK_TYPE = "yolov3"; -const std::string kCALIB_TABLE_PATH = kDS_LIB_PATH + "calibration/yolov3-calibration.table"; -const uint kBBOXES = 3; -const std::vector kANCHORS = {10.0, 13.0, 16.0, 30.0, 33.0, 23.0, 30.0, 61.0, 62.0, - 45.0, 59.0, 119.0, 116.0, 90.0, 156.0, 198.0, 373.0, 326.0}; -#endif - -// Model V3 specific unique global vars -const uint kSTRIDE_1 = 32; -const uint kSTRIDE_2 = 16; -const uint kSTRIDE_3 = 8; -const uint kGRID_SIZE_1 = kINPUT_H / kSTRIDE_1; -const uint kGRID_SIZE_2 = kINPUT_H / kSTRIDE_2; -const uint kGRID_SIZE_3 = kINPUT_H / kSTRIDE_3; -const uint64_t kOUTPUT_SIZE_1 = kGRID_SIZE_1 * kGRID_SIZE_1 * (kBBOXES * (5 + kOUTPUT_CLASSES)); -const uint64_t kOUTPUT_SIZE_2 = kGRID_SIZE_2 * kGRID_SIZE_2 * (kBBOXES * (5 + kOUTPUT_CLASSES)); -const uint64_t kOUTPUT_SIZE_3 = kGRID_SIZE_3 * kGRID_SIZE_3 * (kBBOXES * (5 + kOUTPUT_CLASSES)); -const std::vector kMASK_1 = {6, 7, 8}; -const std::vector kMASK_2 = {3, 4, 5}; -const std::vector kMASK_3 = {0, 1, 2}; -const std::string kOUTPUT_BLOB_NAME_1 = "yolo_83"; -const std::string kOUTPUT_BLOB_NAME_2 = "yolo_95"; -const std::string kOUTPUT_BLOB_NAME_3 = "yolo_107"; - -} // namespace config \ No newline at end of file diff --git a/sources/gst-yoloplugin/yoloplugin_lib/network_config.h b/sources/gst-yoloplugin/yoloplugin_lib/network_config.h deleted file mode 100644 index a6d7020..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/network_config.h +++ /dev/null @@ -1,97 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#ifndef _NETWORK_H_ -#define _NETWORK_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// Uncomment the macro to choose a type of network -#define MODEL_V2 -// #define MODEL_V3 - -namespace config -{ -// Common global vars -extern const bool kPRINT_PRED_INFO; -extern const bool kPRINT_PERF_INFO; -extern const bool kSAVE_DETECTIONS; - -extern const std::string kPRECISION; -extern const std::string kINPUT_BLOB_NAME; -extern const uint kINPUT_H; -extern const uint kINPUT_W; -extern const uint kINPUT_C; -extern const uint64_t kINPUT_SIZE; -extern const uint kOUTPUT_CLASSES; -extern const float kPROB_THRESH; -extern const float kNMS_THRESH; -extern const std::vector kCLASS_NAMES; -extern const uint kBBOXES; -extern const std::vector kANCHORS; - -extern const std::string kMODELS_PATH; -extern const std::string kDETECTION_RESULTS_PATH; -extern const std::string kYOLO_CONFIG_PATH; -extern const std::string kTRAINED_WEIGHTS_PATH; -extern const std::string kNETWORK_TYPE; -extern const std::string kCALIB_TABLE_PATH; -extern const std::string kCALIBRATION_SET; -extern const std::string kTEST_IMAGES; - -// Model V2 specific global vars -extern const uint kSTRIDE; -extern const uint kGRID_SIZE; -extern const uint64_t kOUTPUT_SIZE; -extern const std::string kOUTPUT_BLOB_NAME; - -// Model V3 specific global vars -extern const uint kSTRIDE_1; -extern const uint kSTRIDE_2; -extern const uint kSTRIDE_3; -extern const uint kGRID_SIZE_1; -extern const uint kGRID_SIZE_2; -extern const uint kGRID_SIZE_3; -extern const uint64_t kOUTPUT_SIZE_1; -extern const uint64_t kOUTPUT_SIZE_2; -extern const uint64_t kOUTPUT_SIZE_3; -extern const std::vector kMASK_1; -extern const std::vector kMASK_2; -extern const std::vector kMASK_3; -extern const std::string kOUTPUT_BLOB_NAME_1; -extern const std::string kOUTPUT_BLOB_NAME_2; -extern const std::string kOUTPUT_BLOB_NAME_3; - -} // namespace config - -#endif //_NETWORK_H_ \ No newline at end of file diff --git a/sources/gst-yoloplugin/yoloplugin_lib/plugin_factory.cpp b/sources/gst-yoloplugin/yoloplugin_lib/plugin_factory.cpp deleted file mode 100644 index 32f4511..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/plugin_factory.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#include "plugin_factory.h" - -PluginFactory::PluginFactory() : m_ReorgLayer{nullptr}, m_RegionLayer{nullptr} -{ - for (int i = 0; i < m_MaxLeakyLayers; ++i) m_LeakyReLULayers[i] = nullptr; -} - -nvinfer1::IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialData, - size_t serialLength) -{ - assert(isPlugin(layerName)); - if (std::string(layerName).find("leaky") != std::string::npos) - { - assert(m_LeakyReLUCount >= 0 && m_LeakyReLUCount <= m_MaxLeakyLayers); - assert(m_LeakyReLULayers[m_LeakyReLUCount] == nullptr); - m_LeakyReLULayers[m_LeakyReLUCount] - = unique_ptr_INvPlugin(nvinfer1::plugin::createPReLUPlugin(serialData, serialLength)); - ++m_LeakyReLUCount; - return m_LeakyReLULayers[m_LeakyReLUCount - 1].get(); - } - else if (std::string(layerName).find("reorg") != std::string::npos) - { - assert(m_ReorgLayer == nullptr); - m_ReorgLayer = unique_ptr_INvPlugin( - nvinfer1::plugin::createYOLOReorgPlugin(serialData, serialLength)); - return m_ReorgLayer.get(); - } - else if (std::string(layerName).find("region") != std::string::npos) - { - assert(m_RegionLayer == nullptr); - m_RegionLayer = unique_ptr_INvPlugin( - nvinfer1::plugin::createYOLORegionPlugin(serialData, serialLength)); - return m_RegionLayer.get(); - } - else - { - std::cout << "Unrecognised layer : " << layerName << std::endl; - assert(0); - return nullptr; - } -} - -bool PluginFactory::isPlugin(const char* name) -{ - return ((std::string(name).find("leaky") != std::string::npos) - || (std::string(name).find("reorg") != std::string::npos) - || (std::string(name).find("region") != std::string::npos)); -} - -void PluginFactory::destroy() -{ - m_ReorgLayer.reset(); - m_RegionLayer.reset(); - - for (int i = 0; i < m_MaxLeakyLayers; ++i) - { - m_LeakyReLULayers[i].reset(); - } - - m_LeakyReLUCount = 0; -} \ No newline at end of file diff --git a/sources/gst-yoloplugin/yoloplugin_lib/plugin_factory.h b/sources/gst-yoloplugin/yoloplugin_lib/plugin_factory.h deleted file mode 100644 index 4c8eada..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/plugin_factory.h +++ /dev/null @@ -1,85 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#ifndef __PLUGIN_LAYER_H__ -#define __PLUGIN_LAYER_H__ - -#include -#include -#include -#include -#include - -#include "NvInferPlugin.h" - -#define NV_CUDA_CHECK(status) \ - { \ - if (status != 0) \ - { \ - std::cout << "Cuda failure: " << cudaGetErrorString(status) << " at line " << __LINE__ \ - << std::endl; \ - abort(); \ - } \ - } - -class PluginFactory : public nvinfer1::IPluginFactory -{ - -public: - PluginFactory(); - nvinfer1::IPlugin* createPlugin(const char* layerName, const void* serialData, - size_t serialLength) override; - bool isPlugin(const char* name); - void destroy(); - -private: - static const int m_MaxLeakyLayers = 72; - static const int m_ReorgStride = 2; - static constexpr float m_LeakyNegSlope = 0.1; - static const int m_NumBoxes = 5; - static const int m_NumCoords = 4; - static const int m_NumClasses = 80; - int m_LeakyReLUCount = 0; - nvinfer1::plugin::RegionParameters m_RegionParameters{m_NumBoxes, m_NumCoords, m_NumClasses, - nullptr}; - - struct nvPluginDeleter - { - void operator()(nvinfer1::plugin::INvPlugin* ptr) - { - if (ptr) - { - ptr->destroy(); - } - } - }; - typedef std::unique_ptr unique_ptr_INvPlugin; - - unique_ptr_INvPlugin m_ReorgLayer; - unique_ptr_INvPlugin m_RegionLayer; - unique_ptr_INvPlugin m_LeakyReLULayers[m_MaxLeakyLayers]; -}; - -#endif // __PLUGIN_LAYER_H__ \ No newline at end of file diff --git a/sources/gst-yoloplugin/yoloplugin_lib/trt_utils.cpp b/sources/gst-yoloplugin/yoloplugin_lib/trt_utils.cpp deleted file mode 100644 index 173762d..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/trt_utils.cpp +++ /dev/null @@ -1,649 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#include "trt_utils.h" - -#include -#include -#include - -cv::Mat blobFromDsImages(const std::vector& inputImages, const int& inputH, - const int& inputW) -{ - std::vector letterboxStack(inputImages.size()); - for (uint i = 0; i < inputImages.size(); ++i) - { - inputImages.at(i).getLetterBoxedImage().copyTo(letterboxStack.at(i)); - } - return cv::dnn::blobFromImages(letterboxStack, 1.0, cv::Size(inputW, inputH), - cv::Scalar(0.0, 0.0, 0.0), false, false); -} - -static void leftTrim(std::string& s) -{ - s.erase(s.begin(), find_if(s.begin(), s.end(), [](int ch) { return !isspace(ch); })); -} - -static void rightTrim(std::string& s) -{ - s.erase(find_if(s.rbegin(), s.rend(), [](int ch) { return !isspace(ch); }).base(), s.end()); -} - -std::string trim(std::string s) -{ - leftTrim(s); - rightTrim(s); - return s; -} - -float clamp(const float val, const float minVal, const float maxVal) -{ - assert(minVal <= maxVal); - return std::min(maxVal, std::max(minVal, val)); -} - -bool fileExists(const std::string fileName) -{ - if (!std::experimental::filesystem::exists(std::experimental::filesystem::path(fileName))) - { - std::cout << "File does not exist : " << fileName << std::endl; - return false; - } - return true; -} - -BBox convertBBox(const float& bx, const float& by, const float& bw, const float& bh, - const int& stride) -{ - BBox b; - // Restore coordinates to network input resolution - float x = bx * stride; - float y = by * stride; - - b.x1 = x - bw / 2; - b.x2 = x + bw / 2; - - b.y1 = y - bh / 2; - b.y2 = y + bh / 2; - - return b; -} - -void printPredictions(const BBoxInfo& b, const std::string& className) -{ - std::cout << " label:" << b.label << "(" << className << ")" - << " confidence:" << b.prob << " xmin:" << b.box.x1 << " ymin:" << b.box.y1 - << " xmax:" << b.box.x2 << " ymax:" << b.box.y2 << std::endl; -} - -std::vector loadImageList(const std::string filename) -{ - assert(fileExists(filename)); - std::vector labelInfo; - - FILE* f = fopen(filename.c_str(), "r"); - if (!f) - { - std::cout << "failed to open " << filename; - assert(0); - } - - char str[512]; - while (fgets(str, 512, f) != NULL) - { - for (int i = 0; str[i] != '\0'; ++i) - { - if (str[i] == '\n') - { - str[i] = '\0'; - break; - } - } - labelInfo.push_back(str); - } - fclose(f); - return labelInfo; -} - -std::vector nonMaximumSuppression(const float nmsThresh, std::vector binfo) -{ - auto overlap1D = [](float x1min, float x1max, float x2min, float x2max) -> float { - if (x1min > x2min) - { - std::swap(x1min, x2min); - std::swap(x1max, x2max); - } - return x1max < x2min ? 0 : std::min(x1max, x2max) - x2min; - }; - auto computeIoU = [&overlap1D](BBox& bbox1, BBox& bbox2) -> float { - float overlapX = overlap1D(bbox1.x1, bbox1.x2, bbox2.x1, bbox2.x2); - float overlapY = overlap1D(bbox1.y1, bbox1.y2, bbox2.y1, bbox2.y2); - float area1 = (bbox1.x2 - bbox1.x1) * (bbox1.y2 - bbox1.y1); - float area2 = (bbox2.x2 - bbox2.x1) * (bbox2.y2 - bbox2.y1); - float overlap2D = overlapX * overlapY; - float u = area1 + area2 - overlap2D; - return u == 0 ? 0 : overlap2D / u; - }; - - std::stable_sort(binfo.begin(), binfo.end(), - [](const BBoxInfo& b1, const BBoxInfo& b2) { return b1.prob > b2.prob; }); - std::vector out; - for (auto i : binfo) - { - bool keep = true; - for (auto j : out) - { - if (keep) - { - float overlap = computeIoU(i.box, j.box); - keep = overlap <= nmsThresh; - } - else - break; - } - if (keep) out.push_back(i); - } - return out; -} - -nvinfer1::ICudaEngine* loadTRTEngine(const std::string planFilePath, PluginFactory* pluginFactory) -{ - // reading the model in memory - std::cout << "Loading TRT Engine..." << std::endl; - assert(fileExists(planFilePath)); - std::stringstream trtModelStream; - trtModelStream.seekg(0, trtModelStream.beg); - std::ifstream cache(planFilePath); - assert(cache.good()); - trtModelStream << cache.rdbuf(); - cache.close(); - - // calculating model size - trtModelStream.seekg(0, std::ios::end); - const int modelSize = trtModelStream.tellg(); - trtModelStream.seekg(0, std::ios::beg); - void* modelMem = malloc(modelSize); - trtModelStream.read((char*) modelMem, modelSize); - - Logger nvLogger; - nvinfer1::IRuntime* runtime = nvinfer1::createInferRuntime(nvLogger); - nvinfer1::ICudaEngine* engine - = runtime->deserializeCudaEngine(modelMem, modelSize, pluginFactory); - free(modelMem); - runtime->destroy(); - std::cout << "Loading Complete!" << std::endl; - - return engine; -} - -std::vector> parseConfig(const std::string cfgFilePath) -{ - std::ifstream file(cfgFilePath); - assert(file.good()); - std::string line; - std::vector> blocks; - std::map block; - - while (getline(file, line)) - { - if (line.size() == 0) continue; - if (line.front() == '#') continue; - line = trim(line); - if (line.front() == '[') - { - if (block.size() > 0) - { - blocks.push_back(block); - block.clear(); - } - std::string key = "type"; - std::string value = trim(line.substr(1, line.size() - 2)); - block.insert(std::pair(key, value)); - } - else - { - int cpos = line.find('='); - std::string key = trim(line.substr(0, cpos)); - std::string value = trim(line.substr(cpos + 1)); - block.insert(std::pair(key, value)); - } - } - blocks.push_back(block); - - return blocks; -} - -void displayConfig(const std::vector>& blocks) -{ - for (uint i = 0; i < blocks.size(); ++i) - { - std::map block = blocks.at(i); - std::cout << "--------------------------------" << std::endl; - std::cout << "[ " << block.at("type") << " ]" << std::endl; - for (auto& it : block) - { - if (it.first == "type") continue; - std::cout << it.first << " --> " << it.second << std::endl; - } - std::cout << std::endl; - } -} - -std::vector loadWeights(const std::string weightsFilePath, const std::string& networkType) -{ - std::cout << "Loading pre-trained weights..." << std::endl; - std::ifstream file(weightsFilePath, std::ios_base::binary); - assert(file.good()); - std::string line; - - if (networkType == "yolov2") - { - // Remove 4 int32 bytes of data from the stream belonging to the header - file.ignore(4 * 4); - } - else if (networkType == "yolov3") - { - // Remove 5 int32 bytes of data from the stream belonging to the header - file.ignore(4 * 5); - } - else - { - std::cout << "Invalid network type" << std::endl; - assert(0); - } - - std::vector weights; - char* floatWeight = new char[4]; - while (!file.eof()) - { - file.read(floatWeight, 4); - assert(file.gcount() == 4); - weights.push_back(*reinterpret_cast(floatWeight)); - if (file.peek() == std::istream::traits_type::eof()) break; - } - std::cout << "Loading complete!" << std::endl; - delete[] floatWeight; - - return weights; -} - -std::string dimsToString(const nvinfer1::Dims d) -{ - std::stringstream s; - assert(d.nbDims >= 1); - for (int i = 0; i < d.nbDims - 1; ++i) - { - s << std::setw(4) << d.d[i] << " x"; - } - s << std::setw(4) << d.d[d.nbDims - 1]; - - return s.str(); -} - -void displayDimType(const nvinfer1::Dims d) -{ - std::cout << "(" << d.nbDims << ") "; - for (int i = 0; i < d.nbDims; ++i) - { - switch (d.type[i]) - { - case nvinfer1::DimensionType::kSPATIAL: std::cout << "kSPATIAL "; break; - case nvinfer1::DimensionType::kCHANNEL: std::cout << "kCHANNEL "; break; - case nvinfer1::DimensionType::kINDEX: std::cout << "kINDEX "; break; - case nvinfer1::DimensionType::kSEQUENCE: std::cout << "kSEQUENCE "; break; - } - } - std::cout << std::endl; -} - -int getNumChannels(nvinfer1::ITensor* t) -{ - nvinfer1::Dims d = t->getDimensions(); - assert(d.nbDims == 3); - - return d.d[0]; -} - -nvinfer1::ILayer* netAddMaxpool(int layerIdx, std::map& block, - nvinfer1::ITensor* input, nvinfer1::INetworkDefinition* network) -{ - assert(block.at("type") == "maxpool"); - assert(block.find("size") != block.end()); - assert(block.find("stride") != block.end()); - - int size = std::stoi(block.at("size")); - int stride = std::stoi(block.at("stride")); - - nvinfer1::IPoolingLayer* pool - = network->addPooling(*input, nvinfer1::PoolingType::kMAX, nvinfer1::DimsHW{size, size}); - assert(pool); - std::string maxpoolLayerName = "maxpool_" + std::to_string(layerIdx); - pool->setStride(nvinfer1::DimsHW{stride, stride}); - pool->setName(maxpoolLayerName.c_str()); - - return pool; -} - -nvinfer1::ILayer* netAddConvLinear(int layerIdx, std::map& block, - std::vector& weights, - std::vector& trtWeights, int& weightPtr, - int& inputChannels, nvinfer1::ITensor* input, - nvinfer1::INetworkDefinition* network) -{ - assert(block.at("type") == "convolutional"); - assert(block.find("batch_normalize") == block.end()); - assert(block.at("activation") == "linear"); - assert(block.find("filters") != block.end()); - assert(block.find("pad") != block.end()); - assert(block.find("size") != block.end()); - assert(block.find("stride") != block.end()); - - int filters = std::stoi(block.at("filters")); - int padding = std::stoi(block.at("pad")); - int kernelSize = std::stoi(block.at("size")); - int stride = std::stoi(block.at("stride")); - int pad; - if (padding) - pad = (kernelSize - 1) / 2; - else - pad = 0; - // load the convolution layer bias - nvinfer1::Weights convBias{nvinfer1::DataType::kFLOAT, nullptr, filters}; - float* val = new float[filters]; - for (int i = 0; i < filters; ++i) - { - val[i] = weights[weightPtr]; - weightPtr++; - } - convBias.values = val; - trtWeights.push_back(convBias); - // load the convolutional layer weights - int size = filters * inputChannels * kernelSize * kernelSize; - nvinfer1::Weights convWt{nvinfer1::DataType::kFLOAT, nullptr, size}; - val = new float[size]; - for (int i = 0; i < size; ++i) - { - val[i] = weights[weightPtr]; - weightPtr++; - } - convWt.values = val; - trtWeights.push_back(convWt); - nvinfer1::IConvolutionLayer* conv = network->addConvolution( - *input, filters, nvinfer1::DimsHW{kernelSize, kernelSize}, convWt, convBias); - assert(conv != nullptr); - std::string convLayerName = "conv_" + std::to_string(layerIdx); - conv->setName(convLayerName.c_str()); - conv->setStride(nvinfer1::DimsHW{stride, stride}); - conv->setPadding(nvinfer1::DimsHW{pad, pad}); - inputChannels = filters; - - return conv; -} - -nvinfer1::ILayer* netAddConvBNLeaky(int layerIdx, std::map& block, - std::vector& weights, - std::vector& trtWeights, int& weightPtr, - int& inputChannels, nvinfer1::ITensor* input, - nvinfer1::INetworkDefinition* network) -{ - assert(block.at("type") == "convolutional"); - assert(block.find("batch_normalize") != block.end()); - assert(block.at("batch_normalize") == "1"); - assert(block.at("activation") == "leaky"); - assert(block.find("filters") != block.end()); - assert(block.find("pad") != block.end()); - assert(block.find("size") != block.end()); - assert(block.find("stride") != block.end()); - - bool batchNormalize, bias; - if (block.find("batch_normalize") != block.end()) - { - batchNormalize = (block.at("batch_normalize") == "1"); - bias = false; - } - else - { - batchNormalize = false; - bias = true; - } - // all conv_bn_leaky layers assume bias is false - assert(batchNormalize == true && bias == false); - - int filters = std::stoi(block.at("filters")); - int padding = std::stoi(block.at("pad")); - int kernelSize = std::stoi(block.at("size")); - int stride = std::stoi(block.at("stride")); - int pad; - if (padding) - pad = (kernelSize - 1) / 2; - else - pad = 0; - - /***** CONVOLUTION LAYER *****/ - /*****************************/ - // batch norm weights are before the conv layer - // load BN biases (bn_biases) - std::vector bnBiases; - for (int i = 0; i < filters; ++i) - { - bnBiases.push_back(weights[weightPtr]); - weightPtr++; - } - // load BN weights - std::vector bnWeights; - for (int i = 0; i < filters; ++i) - { - bnWeights.push_back(weights[weightPtr]); - weightPtr++; - } - // load BN running_mean - std::vector bnRunningMean; - for (int i = 0; i < filters; ++i) - { - bnRunningMean.push_back(weights[weightPtr]); - weightPtr++; - } - // load BN running_var - std::vector bnRunningVar; - for (int i = 0; i < filters; ++i) - { - // 1e-05 for numerical stability - bnRunningVar.push_back(sqrt(weights[weightPtr] + 1.0e-5)); - weightPtr++; - } - // load Conv layer weights (GKCRS) - int size = filters * inputChannels * kernelSize * kernelSize; - nvinfer1::Weights convWt{nvinfer1::DataType::kFLOAT, nullptr, size}; - float* val = new float[size]; - for (int i = 0; i < size; ++i) - { - val[i] = weights[weightPtr]; - weightPtr++; - } - convWt.values = val; - trtWeights.push_back(convWt); - nvinfer1::Weights convBias{nvinfer1::DataType::kFLOAT, nullptr, 0}; - trtWeights.push_back(convBias); - nvinfer1::IConvolutionLayer* conv = network->addConvolution( - *input, filters, nvinfer1::DimsHW{kernelSize, kernelSize}, convWt, convBias); - assert(conv != nullptr); - std::string convLayerName = "conv_" + std::to_string(layerIdx); - conv->setName(convLayerName.c_str()); - conv->setStride(nvinfer1::DimsHW{stride, stride}); - conv->setPadding(nvinfer1::DimsHW{pad, pad}); - inputChannels = filters; - - /***** BATCHNORM LAYER *****/ - /***************************/ - size = filters; - // create the weights - nvinfer1::Weights shift{nvinfer1::DataType::kFLOAT, nullptr, size}; - nvinfer1::Weights scale{nvinfer1::DataType::kFLOAT, nullptr, size}; - nvinfer1::Weights power{nvinfer1::DataType::kFLOAT, nullptr, size}; - float* shiftWt = new float[size]; - for (int i = 0; i < size; ++i) - { - shiftWt[i] - = bnBiases.at(i) - ((bnRunningMean.at(i) * bnWeights.at(i)) / bnRunningVar.at(i)); - } - shift.values = shiftWt; - float* scaleWt = new float[size]; - for (int i = 0; i < size; ++i) - { - scaleWt[i] = bnWeights.at(i) / bnRunningVar[i]; - } - scale.values = scaleWt; - float* powerWt = new float[size]; - for (int i = 0; i < size; ++i) - { - powerWt[i] = 1.0; - } - power.values = powerWt; - trtWeights.push_back(shift); - trtWeights.push_back(scale); - trtWeights.push_back(power); - // Add the batch norm layers - nvinfer1::IScaleLayer* bn = network->addScale( - *conv->getOutput(0), nvinfer1::ScaleMode::kCHANNEL, shift, scale, power); - assert(bn != nullptr); - std::string bnLayerName = "batch_norm_" + std::to_string(layerIdx); - bn->setName(bnLayerName.c_str()); - /***** ACTIVATION LAYER *****/ - /****************************/ - nvinfer1::IPlugin* leakyRELU = nvinfer1::plugin::createPReLUPlugin(0.1); - assert(leakyRELU != nullptr); - nvinfer1::ITensor* bnOutput = bn->getOutput(0); - nvinfer1::IPluginLayer* leaky = network->addPlugin(&bnOutput, 1, *leakyRELU); - assert(leaky != nullptr); - std::string leakyLayerName = "leaky_" + std::to_string(layerIdx); - leaky->setName(leakyLayerName.c_str()); - - return leaky; -} - -nvinfer1::ILayer* netAddUpsample(int layerIdx, std::map& block, - std::vector& weights, int& inputChannels, - nvinfer1::ITensor* input, nvinfer1::INetworkDefinition* network) -{ - assert(block.at("type") == "upsample"); - assert(block.at("stride") == "2"); - nvinfer1::Dims inpDims = input->getDimensions(); - assert(inpDims.nbDims == 3); - int h = inpDims.d[1]; - int w = inpDims.d[2]; - // add pre multiply matrix as a constant - nvinfer1::Dims preDims{3, - {1, 2 * h, w}, - {nvinfer1::DimensionType::kCHANNEL, nvinfer1::DimensionType::kSPATIAL, - nvinfer1::DimensionType::kSPATIAL}}; - int size = 2 * h * w; - nvinfer1::Weights pre{nvinfer1::DataType::kFLOAT, nullptr, size}; - float* preWt = new float[size]; - /* (2*h * w) - [ [1, 0, ..., 0], - [1, 0, ..., 0], - [0, 1, ..., 0], - [0, 1, ..., 0], - ..., - ..., - [0, 0, ..., 1], - [0, 0, ..., 1] ] - */ - for (int i = 0, idx = 0; i < h; ++i) - { - for (int j = 0; j < w; ++j, ++idx) - { - preWt[idx] = (i == j) ? 1.0 : 0.0; - } - for (int j = 0; j < w; ++j, ++idx) - { - preWt[idx] = (i == j) ? 1.0 : 0.0; - } - } - pre.values = preWt; - nvinfer1::IConstantLayer* preM = network->addConstant(preDims, pre); - assert(preM != nullptr); - std::string preLayerName = "pre_" + std::to_string(layerIdx); - preM->setName(preLayerName.c_str()); - // add post multiply matrix as a constant - nvinfer1::Dims postDims{3, - {1, h, 2 * w}, - {nvinfer1::DimensionType::kCHANNEL, nvinfer1::DimensionType::kSPATIAL, - nvinfer1::DimensionType::kSPATIAL}}; - size = 2 * h * w; - nvinfer1::Weights post{nvinfer1::DataType::kFLOAT, nullptr, size}; - float* postWt = new float[size]; - /* (h * 2*w) - [ [1, 1, 0, 0, ..., 0, 0], - [0, 0, 1, 1, ..., 0, 0], - ..., - ..., - [0, 0, 0, 0, ..., 1, 1] ] - */ - for (int i = 0, idx = 0; i < h; ++i) - { - for (int j = 0; j < 2 * w; ++j, ++idx) - { - postWt[idx] = (j / 2 == i) ? 1.0 : 0.0; - } - } - post.values = postWt; - nvinfer1::IConstantLayer* post_m = network->addConstant(postDims, post); - assert(post_m != nullptr); - std::string postLayerName = "post_" + std::to_string(layerIdx); - post_m->setName(postLayerName.c_str()); - // add matrix multiply layers for upsampling - nvinfer1::IMatrixMultiplyLayer* mm1 - = network->addMatrixMultiply(*preM->getOutput(0), false, *input, false); - assert(mm1 != nullptr); - std::string mm1LayerName = "mm1_" + std::to_string(layerIdx); - mm1->setName(mm1LayerName.c_str()); - nvinfer1::IMatrixMultiplyLayer* mm2 - = network->addMatrixMultiply(*mm1->getOutput(0), false, *post_m->getOutput(0), false); - assert(mm2 != nullptr); - std::string mm2LayerName = "mm2_" + std::to_string(layerIdx); - mm2->setName(mm2LayerName.c_str()); - // switch dimension **types** from kSPATIAL, kCHANNEL, kSPATIAL to kCHANNEL, kSPATIAL, kSPATIAL - nvinfer1::Dims outDims{3, - {inputChannels, 2 * h, 2 * w}, - {nvinfer1::DimensionType::kCHANNEL, nvinfer1::DimensionType::kSPATIAL, - nvinfer1::DimensionType::kSPATIAL}}; - nvinfer1::IShuffleLayer* reshape = network->addShuffle(*mm2->getOutput(0)); - assert(reshape != nullptr); - std::string reshapeLayerName = "upsample_" + std::to_string(layerIdx); - reshape->setName(reshapeLayerName.c_str()); - reshape->setReshapeDimensions(outDims); - - return reshape; -} - -void printLayerInfo(std::string layerIndex, std::string layerName, std::string layerInput, - std::string layerOutput, std::string weightPtr) -{ - std::cout << std::setw(6) << std::left << layerIndex << std::setw(9) << std::left << layerName; - std::cout << std::setw(20) << std::left << layerInput << std::setw(20) << std::left - << layerOutput; - std::cout << std::setw(6) << std::left << weightPtr << std::endl; -} diff --git a/sources/gst-yoloplugin/yoloplugin_lib/trt_utils.h b/sources/gst-yoloplugin/yoloplugin_lib/trt_utils.h deleted file mode 100644 index 1173f57..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/trt_utils.h +++ /dev/null @@ -1,115 +0,0 @@ - -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#ifndef __TRT_UTILS_H__ -#define __TRT_UTILS_H__ - -/* OpenCV headers */ -#include -#include -#include -#include -#include - -#include "NvInfer.h" - -#include "ds_image.h" -#include "plugin_factory.h" - -class DsImage; -struct BBox -{ - float x1, y1, x2, y2; -}; - -struct BBoxInfo -{ - BBox box; - int label; - float prob; -}; - -class Logger : public nvinfer1::ILogger -{ -public: - void log(nvinfer1::ILogger::Severity severity, const char* msg) override - { - // suppress info-level messages - if (severity == Severity::kINFO) return; - - switch (severity) - { - case Severity::kINTERNAL_ERROR: std::cerr << "INTERNAL_ERROR: "; break; - case Severity::kERROR: std::cerr << "ERROR: "; break; - case Severity::kWARNING: std::cerr << "WARNING: "; break; - case Severity::kINFO: std::cerr << "INFO: "; break; - default: std::cerr << "UNKNOWN: "; break; - } - std::cerr << msg << std::endl; - } -}; - -inline float sigmoid(const float& x) { return 1.0f / (1.0f + exp(-x)); } - -// Common helper functions -cv::Mat blobFromDsImages(const std::vector& inputImages, const int& inputH, - const int& inputW); -std::string trim(std::string s); -float clamp(const float val, const float minVal, const float maxVal); -bool fileExists(const std::string fileName); -BBox convertBBox(const float& bx, const float& by, const float& bw, const float& bh, - const int& stride); -void printPredictions(const BBoxInfo& info, const std::string& className); -std::vector loadImageList(const std::string filename); -std::vector nonMaximumSuppression(const float nmsThresh, std::vector binfo); -nvinfer1::ICudaEngine* loadTRTEngine(const std::string planFilePath, PluginFactory* pluginFactory); -std::vector> parseConfig(const std::string cfgFilePath); -void displayConfig(const std::vector>& blocks); -std::vector loadWeights(const std::string weightsFilePath, const std::string& networkType); -std::string dimsToString(const nvinfer1::Dims d); -void displayDimType(const nvinfer1::Dims d); -int getNumChannels(nvinfer1::ITensor* t); - -// Helper functions to create yolo engine -nvinfer1::ILayer* netAddMaxpool(int layerIdx, std::map& block, - nvinfer1::ITensor* input, nvinfer1::INetworkDefinition* network); -nvinfer1::ILayer* netAddConvLinear(int layerIdx, std::map& block, - std::vector& weights, - std::vector& trtWeights, int& weightPtr, - int& inputChannels, nvinfer1::ITensor* input, - nvinfer1::INetworkDefinition* network); -nvinfer1::ILayer* netAddConvBNLeaky(int layerIdx, std::map& block, - std::vector& weights, - std::vector& trtWeights, int& weightPtr, - int& inputChannels, nvinfer1::ITensor* input, - nvinfer1::INetworkDefinition* network); -nvinfer1::ILayer* netAddUpsample(int layerIdx, std::map& block, - std::vector& weights, int& inputChannels, - nvinfer1::ITensor* input, nvinfer1::INetworkDefinition* network); -void printLayerInfo(std::string layerIndex, std::string layerName, std::string layerInput, - std::string layerOutput, std::string weightPtr); - -#endif diff --git a/sources/gst-yoloplugin/yoloplugin_lib/yolo.cpp b/sources/gst-yoloplugin/yoloplugin_lib/yolo.cpp deleted file mode 100644 index c512783..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/yolo.cpp +++ /dev/null @@ -1,397 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#include "yolo.h" -#include "network_config.h" - -Yolo::Yolo(uint batchSize) : - m_ModelsPath(config::kMODELS_PATH), - m_ConfigFilePath(config::kYOLO_CONFIG_PATH), - m_TrainedWeightsPath(config::kTRAINED_WEIGHTS_PATH), - m_NetworkType(config::kNETWORK_TYPE), - m_CalibImagesFilePath(config::kCALIBRATION_SET), - m_CalibTableFilePath(config::kCALIB_TABLE_PATH), - m_Precision(config::kPRECISION), - m_InputBlobName(config::kINPUT_BLOB_NAME), - m_InputH(config::kINPUT_H), - m_InputW(config::kINPUT_W), - m_InputC(config::kINPUT_C), - m_InputSize(config::kINPUT_SIZE), - m_NumOutputClasses(config::kOUTPUT_CLASSES), - m_NumBBoxes(config::kBBOXES), - m_ProbThresh(config::kPROB_THRESH), - m_NMSThresh(config::kNMS_THRESH), - m_Anchors(config::kANCHORS), - m_ClassNames(config::kCLASS_NAMES), - m_PrintPerfInfo(config::kPRINT_PERF_INFO), - m_PrintPredictions(config::kPRINT_PRED_INFO), - m_BatchSize(batchSize), - m_Engine(nullptr), - m_Context(nullptr), - m_Bindings(), - m_TrtOutputBuffers(), - m_InputIndex(-1), - m_CudaStream(nullptr), - m_PluginFactory(new PluginFactory) -{ - std::string planFilePath = m_ModelsPath + m_NetworkType + "-" + m_Precision + "-batch" - + std::to_string(m_BatchSize) + ".engine"; - // Create and cache the engine if not already present - if (!fileExists(planFilePath)) - { - std::cout << "Unable to find cached TensorRT engine for network : " << m_NetworkType - << " precision : " << m_Precision << " and batch size :" << m_BatchSize - << std::endl; - std::cout << "Creating a new TensorRT Engine" << std::endl; - - if (m_Precision == "kFLOAT") - { - createYOLOEngine(m_BatchSize, m_ConfigFilePath, m_TrainedWeightsPath, planFilePath); - } - else if (m_Precision == "kINT8") - { - Int8EntropyCalibrator calibrator(m_BatchSize, m_CalibImagesFilePath, - m_CalibTableFilePath, m_InputSize, m_InputH, m_InputW, - m_InputBlobName); - createYOLOEngine(m_BatchSize, m_ConfigFilePath, m_TrainedWeightsPath, planFilePath, - nvinfer1::DataType::kINT8, &calibrator); - } - else if (m_Precision == "kHALF") - { - createYOLOEngine(m_BatchSize, m_ConfigFilePath, m_TrainedWeightsPath, planFilePath, - nvinfer1::DataType::kHALF, nullptr); - } - else - { - std::cout << "Unrecognized precision type " << m_Precision << std::endl; - assert(0); - } - } - else - std::cout << "Using previously generated plan file located at " << planFilePath - << std::endl; - - assert(m_PluginFactory != nullptr); - m_Engine = loadTRTEngine(planFilePath, m_PluginFactory); - assert(m_Engine != nullptr); - m_Context = m_Engine->createExecutionContext(); - assert(m_Context != nullptr); - m_Bindings.resize(m_Engine->getNbBindings(), nullptr); - m_TrtOutputBuffers.resize(m_Engine->getNbBindings() - 1, nullptr); - m_InputIndex = m_Engine->getBindingIndex(m_InputBlobName.c_str()); - assert(m_InputIndex != -1); - assert(m_BatchSize <= static_cast(m_Engine->getMaxBatchSize())); - NV_CUDA_CHECK(cudaStreamCreate(&m_CudaStream)); -}; - -Yolo::~Yolo() -{ - for (auto buffer : m_TrtOutputBuffers) delete[] buffer; - for (auto binding : m_Bindings) NV_CUDA_CHECK(cudaFree(binding)); - cudaStreamDestroy(m_CudaStream); - if (m_Context) - { - m_Context->destroy(); - m_Context = nullptr; - } - - if (m_Engine) - { - m_Engine->destroy(); - m_Engine = nullptr; - } - - m_PluginFactory->destroy(); -} - -void Yolo::createYOLOEngine(const int batchSize, const std::string yoloConfigPath, - const std::string trainedWeightsPath, const std::string planFilePath, - const nvinfer1::DataType dataType, Int8EntropyCalibrator* calibrator) -{ - assert(fileExists(yoloConfigPath)); - assert(fileExists(trainedWeightsPath)); - - std::vector> blocks = parseConfig(yoloConfigPath); - std::vector weights = loadWeights(trainedWeightsPath, m_NetworkType); - std::vector trtWeights; - int weightPtr = 0; - int channels = m_InputC; - Logger nvLogger; - nvinfer1::IBuilder* builder = nvinfer1::createInferBuilder(nvLogger); - nvinfer1::INetworkDefinition* network = builder->createNetwork(); - - if ((dataType == nvinfer1::DataType::kINT8 && !builder->platformHasFastInt8()) - || (dataType == nvinfer1::DataType::kHALF && !builder->platformHasFastFp16())) - { - std::cout << "Platform doesn't support this precision." << std::endl; - assert(0); - } - - nvinfer1::ITensor* data = network->addInput(m_InputBlobName.c_str(), nvinfer1::DataType::kFLOAT, - nvinfer1::DimsCHW{static_cast(m_InputC), - static_cast(m_InputH), - static_cast(m_InputW)}); - nvinfer1::ITensor* previous = data; - std::vector tensorOutputs; - std::vector outputLayers; - - // build the network using the network API - for (uint i = 0; i < blocks.size(); ++i) - { - // check if num. of channels is correct - assert(getNumChannels(previous) == channels); - std::string layerIndex = "(" + std::to_string(i) + ")"; - - if (blocks.at(i).at("type") == "net") - { - printLayerInfo("", "layer", " inp_size", " out_size", "weightPtr"); - } - else if (blocks.at(i).at("type") == "convolutional") - { - std::string inputVol = dimsToString(previous->getDimensions()); - nvinfer1::ILayer* out; - // check if batch_norm enabled - if (blocks.at(i).find("batch_normalize") != blocks.at(i).end()) - out = netAddConvBNLeaky(i, blocks.at(i), weights, trtWeights, weightPtr, channels, - previous, network); - else - out = netAddConvLinear(i, blocks.at(i), weights, trtWeights, weightPtr, channels, - previous, network); - previous = out->getOutput(0); - assert(previous != nullptr); - std::string outputVol = dimsToString(previous->getDimensions()); - tensorOutputs.push_back(out->getOutput(0)); - printLayerInfo(layerIndex, "conv", inputVol, outputVol, std::to_string(weightPtr)); - } - else if (blocks.at(i).at("type") == "shortcut") - { - assert(blocks.at(i).at("activation") == "linear"); - assert(blocks.at(i).find("from") != blocks.at(i).end()); - int from = stoi(blocks.at(i).at("from")); - - std::string inputVol = dimsToString(previous->getDimensions()); - // check if indexes are correct - assert((i - 2 >= 0) && (i - 2 < tensorOutputs.size())); - assert((i + from - 1 >= 0) && (i + from - 1 < tensorOutputs.size())); - assert(i + from - 1 < i - 2); - nvinfer1::IElementWiseLayer* ew - = network->addElementWise(*tensorOutputs[i - 2], *tensorOutputs[i + from - 1], - nvinfer1::ElementWiseOperation::kSUM); - assert(ew != nullptr); - std::string ewLayerName = "shortcut_" + std::to_string(i); - ew->setName(ewLayerName.c_str()); - previous = ew->getOutput(0); - assert(previous != nullptr); - std::string outputVol = dimsToString(previous->getDimensions()); - tensorOutputs.push_back(ew->getOutput(0)); - printLayerInfo(layerIndex, "skip", inputVol, outputVol, " -"); - } - else if (blocks.at(i).at("type") == "yolo") - { - std::string layerName = "yolo_" + std::to_string(i); - previous->setName(layerName.c_str()); - network->markOutput(*previous); - std::string inputVol = dimsToString(previous->getDimensions()); - // for index consistency - tensorOutputs.push_back(tensorOutputs.back()); - outputLayers.push_back(tensorOutputs.back()); - printLayerInfo(layerIndex, "yolo", inputVol, " -", " -"); - } - else if (blocks.at(i).at("type") == "region") - { - nvinfer1::plugin::RegionParameters RegionParameters{ - static_cast(m_NumBBoxes), 4, static_cast(m_NumOutputClasses), nullptr}; - std::string inputVol = dimsToString(previous->getDimensions()); - nvinfer1::IPlugin* regionPlugin - = nvinfer1::plugin::createYOLORegionPlugin(RegionParameters); - assert(regionPlugin != nullptr); - nvinfer1::IPluginLayer* region = network->addPlugin(&previous, 1, *regionPlugin); - assert(region != nullptr); - - std::string layerName = "region_" + std::to_string(i); - region->setName(layerName.c_str()); - - previous = region->getOutput(0); - previous->setName(layerName.c_str()); - assert(previous != nullptr); - std::string outputVol = dimsToString(previous->getDimensions()); - network->markOutput(*previous); - channels = getNumChannels(previous); - // for index consistency - tensorOutputs.push_back(region->getOutput(0)); - outputLayers.push_back(tensorOutputs.back()); - printLayerInfo(layerIndex, "region", inputVol, outputVol, std::to_string(weightPtr)); - } - else if (blocks.at(i).at("type") == "reorg") - { - std::string inputVol = dimsToString(previous->getDimensions()); - nvinfer1::IPlugin* reorgPlugin = nvinfer1::plugin::createYOLOReorgPlugin(2); - assert(reorgPlugin != nullptr); - nvinfer1::IPluginLayer* reorg = network->addPlugin(&previous, 1, *reorgPlugin); - assert(reorg != nullptr); - - std::string layerName = "reorg_" + std::to_string(i); - reorg->setName(layerName.c_str()); - previous = reorg->getOutput(0); - assert(previous != nullptr); - std::string outputVol = dimsToString(previous->getDimensions()); - channels = getNumChannels(previous); - // for index consistency - tensorOutputs.push_back(reorg->getOutput(0)); - printLayerInfo(layerIndex, "reorg", inputVol, outputVol, std::to_string(weightPtr)); - } - // route layers (single or concat) - else if (blocks.at(i).at("type") == "route") - { - size_t found = blocks.at(i).at("layers").find(","); - if (found != std::string::npos) - { - int idx1 = std::stoi(trim(blocks.at(i).at("layers").substr(0, found))); - int idx2 = std::stoi(trim(blocks.at(i).at("layers").substr(found + 1))); - if (idx1 < 0) - { - idx1 = tensorOutputs.size() + idx1; - } - if (idx2 < 0) - { - idx2 = tensorOutputs.size() + idx2; - } - assert(idx1 < static_cast(tensorOutputs.size()) && idx1 >= 0); - assert(idx2 < static_cast(tensorOutputs.size()) && idx2 >= 0); - nvinfer1::ITensor** concatInputs - = reinterpret_cast(malloc(sizeof(nvinfer1::ITensor*) * 2)); - concatInputs[0] = tensorOutputs[idx1]; - concatInputs[1] = tensorOutputs[idx2]; - nvinfer1::IConcatenationLayer* concat = network->addConcatenation(concatInputs, 2); - assert(concat != nullptr); - std::string concatLayerName = "route_" + std::to_string(i - 1); - concat->setName(concatLayerName.c_str()); - // concatenate along the channel dimension - concat->setAxis(0); - previous = concat->getOutput(0); - assert(previous != nullptr); - std::string outputVol = dimsToString(previous->getDimensions()); - // set the output volume depth - channels - = getNumChannels(tensorOutputs[idx1]) + getNumChannels(tensorOutputs[idx2]); - tensorOutputs.push_back(concat->getOutput(0)); - printLayerInfo(layerIndex, "route", " -", outputVol, - std::to_string(weightPtr)); - } - else - { - int idx = std::stoi(trim(blocks.at(i).at("layers"))); - if (idx < 0) - { - idx = tensorOutputs.size() + idx; - } - assert(idx < static_cast(tensorOutputs.size()) && idx >= 0); - previous = tensorOutputs[idx]; - assert(previous != nullptr); - std::string outputVol = dimsToString(previous->getDimensions()); - // set the output volume depth - channels = getNumChannels(tensorOutputs[idx]); - tensorOutputs.push_back(tensorOutputs[idx]); - printLayerInfo(layerIndex, "route", " -", outputVol, - std::to_string(weightPtr)); - } - } - else if (blocks.at(i).at("type") == "upsample") - { - std::string inputVol = dimsToString(previous->getDimensions()); - nvinfer1::ILayer* out - = netAddUpsample(i - 1, blocks[i], weights, channels, previous, network); - previous = out->getOutput(0); - std::string outputVol = dimsToString(previous->getDimensions()); - tensorOutputs.push_back(out->getOutput(0)); - printLayerInfo(layerIndex, "upsample", inputVol, outputVol, " -"); - } - else if (blocks.at(i).at("type") == "maxpool") - { - std::string inputVol = dimsToString(previous->getDimensions()); - nvinfer1::ILayer* out = netAddMaxpool(i, blocks.at(i), previous, network); - previous = out->getOutput(0); - assert(previous != nullptr); - std::string outputVol = dimsToString(previous->getDimensions()); - tensorOutputs.push_back(out->getOutput(0)); - printLayerInfo(layerIndex, "maxpool", inputVol, outputVol, std::to_string(weightPtr)); - } - else - { - std::cout << "Unsupported layer type --> \"" << blocks.at(i).at("type") << "\"" - << std::endl; - assert(0); - } - } - - std::cout << "Output layers :" << std::endl; - for (auto layer : outputLayers) std::cout << layer->getName() << std::endl; - - builder->setMaxBatchSize(batchSize); - builder->setMaxWorkspaceSize(1 << 20); - - if (dataType == nvinfer1::DataType::kINT8) - { - assert((calibrator != nullptr) && "Invalid calibrator for INT8 precision"); - builder->setInt8Mode(true); - builder->setInt8Calibrator(calibrator); - } - else if (dataType == nvinfer1::DataType::kHALF) - { - builder->setHalf2Mode(true); - } - - // Build the engine - std::cout << "Building the TensorRT Engine..." << std::endl; - nvinfer1::ICudaEngine* engine = builder->buildCudaEngine(*network); - assert(engine != nullptr); - std::cout << "Building complete!" << std::endl; - - // Serialize the engine - std::cout << "Serializing the TensorRT Engine..." << std::endl; - nvinfer1::IHostMemory* modelStream = engine->serialize(); - assert(modelStream != nullptr); - - // write data to output file - std::stringstream gieModelStream; - gieModelStream.seekg(0, gieModelStream.beg); - gieModelStream.write(static_cast(modelStream->data()), modelStream->size()); - std::ofstream outFile; - outFile.open(planFilePath); - outFile << gieModelStream.rdbuf(); - outFile.close(); - - std::cout << "Serialized plan file cached at location : " << planFilePath << std::endl; - network->destroy(); - engine->destroy(); - builder->destroy(); - modelStream->destroy(); - - // deallocate the weights - for (uint i = 0; i < trtWeights.size(); ++i) - { - free(const_cast(trtWeights[i].values)); - } -} \ No newline at end of file diff --git a/sources/gst-yoloplugin/yoloplugin_lib/yolo.h b/sources/gst-yoloplugin/yoloplugin_lib/yolo.h deleted file mode 100644 index 765a6ae..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/yolo.h +++ /dev/null @@ -1,96 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#ifndef _YOLO_H_ -#define _YOLO_H_ - -#include "calibrator.h" -#include "plugin_factory.h" -#include "trt_utils.h" - -#include "NvInfer.h" - -#include -#include -#include - -class Yolo -{ -public: - std::string getNetworkType() const { return m_NetworkType; } - float getNMSThresh() const { return m_NMSThresh; } - std::string getClassName(const int& label) const { return m_ClassNames.at(label); } - std::string getCalibTableFilePath() const { return m_CalibTableFilePath; } - int getInputH() const { return m_InputH; } - int getInputW() const { return m_InputW; } - bool isPrintPredictions() const { return m_PrintPredictions; } - bool isPrintPerfInfo() const { return m_PrintPerfInfo; } - virtual void doInference(const unsigned char* input) = 0; - virtual std::vector decodeDetections(const int& imageIdx, const int& imageH, - const int& imageW) - = 0; - virtual ~Yolo(); - -protected: - explicit Yolo(const uint batchSize); - const std::string m_ModelsPath; - const std::string m_ConfigFilePath; - const std::string m_TrainedWeightsPath; - const std::string m_NetworkType; - const std::string m_CalibImagesFilePath; - const std::string m_CalibTableFilePath; - const std::string m_Precision; - const std::string m_InputBlobName; - const uint m_InputH; - const uint m_InputW; - const uint m_InputC; - const uint64_t m_InputSize; - const uint m_NumOutputClasses; - const uint m_NumBBoxes; - const float m_ProbThresh; - const float m_NMSThresh; - const std::vector m_Anchors; - const std::vector m_ClassNames; - const bool m_PrintPerfInfo; - const bool m_PrintPredictions; - - // TRT specific members - const uint m_BatchSize; - nvinfer1::ICudaEngine* m_Engine; - nvinfer1::IExecutionContext* m_Context; - std::vector m_Bindings; - std::vector m_TrtOutputBuffers; - int m_InputIndex; - cudaStream_t m_CudaStream; - PluginFactory* m_PluginFactory; - -private: - void createYOLOEngine(const int batchSize, const std::string yoloConfigPath, - const std::string trainedWeightsPath, const std::string planFilePath, - const nvinfer1::DataType dataType = nvinfer1::DataType::kFLOAT, - Int8EntropyCalibrator* calibrator = nullptr); -}; - -#endif // _YOLO_H_ \ No newline at end of file diff --git a/sources/gst-yoloplugin/yoloplugin_lib/yoloplugin_lib.cpp b/sources/gst-yoloplugin/yoloplugin_lib/yoloplugin_lib.cpp deleted file mode 100644 index 98edc29..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/yoloplugin_lib.cpp +++ /dev/null @@ -1,182 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#include "yoloplugin_lib.h" -#include "network_config.h" - -#ifdef MODEL_V2 -#include "yolov2.h" -#endif - -#ifdef MODEL_V3 -#include "yolov3.h" -#endif - -#include -#include - -static void decodeBatchDetections(const YoloPluginCtx* ctx, std::vector& outputs) -{ - for (int p = 0; p < ctx->batchSize; ++p) - { - YoloPluginOutput* out = new YoloPluginOutput; - std::vector binfo = ctx->inferenceNetwork->decodeDetections( - p, ctx->initParams.processingHeight, ctx->initParams.processingWidth); - - std::vector remaining - = nonMaximumSuppression(ctx->inferenceNetwork->getNMSThresh(), binfo); - out->numObjects = remaining.size(); - assert(out->numObjects <= MAX_OBJECTS_PER_FRAME); - for (uint j = 0; j < remaining.size(); ++j) - { - BBoxInfo b = remaining.at(j); - YoloPluginObject obj; - obj.left = static_cast(b.box.x1); - obj.top = static_cast(b.box.y1); - obj.width = static_cast(b.box.x2 - b.box.x1); - obj.height = static_cast(b.box.y2 - b.box.y1); - strcpy(obj.label, ctx->inferenceNetwork->getClassName(b.label).c_str()); - out->object[j] = obj; - - if (ctx->inferenceNetwork->isPrintPredictions()) - { - printPredictions(b, ctx->inferenceNetwork->getClassName(b.label)); - } - } - outputs.at(p) = out; - } -} - -static void dsPreProcessBatchInput(const std::vector& cvmats, cv::Mat& batchBlob, - const int& processingHeight, const int& processingWidth, - const int& inputH, const int& inputW) -{ - - std::vector batch_images( - cvmats.size(), cv::Mat(cv::Size(processingWidth, processingHeight), CV_8UC3)); - for (uint i = 0; i < cvmats.size(); ++i) - { - cv::Mat imageResize, imageBorder, imageFloat, inputImage; - inputImage = *cvmats.at(i); - int maxBorder = std::max(inputImage.size().width, inputImage.size().height); - - assert((maxBorder - inputImage.size().height) % 2 == 0); - assert((maxBorder - inputImage.size().width) % 2 == 0); - - int yOffset = (maxBorder - inputImage.size().height) / 2; - int xOffset = (maxBorder - inputImage.size().width) / 2; - - // Letterbox and resize to maintain aspect ratio - cv::copyMakeBorder(inputImage, imageBorder, yOffset, yOffset, xOffset, xOffset, - cv::BORDER_CONSTANT, cv::Scalar(127.5, 127.5, 127.5)); - cv::resize(imageBorder, imageResize, cv::Size(inputW, inputH), 0, 0, cv::INTER_CUBIC); - imageResize.convertTo(imageFloat, CV_32FC3, 1 / 255.0); - batch_images.at(i) = imageFloat; - } - - batchBlob = cv::dnn::blobFromImages(batch_images, 1.0, cv::Size(inputW, inputH), - cv::Scalar(0.0, 0.0, 0.0), false, false); -} - -YoloPluginCtx* YoloPluginCtxInit(YoloPluginInitParams* initParams, size_t batchSize) -{ - YoloPluginCtx* ctx = new YoloPluginCtx; - ctx->initParams = *initParams; - ctx->batchSize = batchSize; - assert(ctx->batchSize > 0); - -#ifdef MODEL_V2 - ctx->inferenceNetwork = new YoloV2(batchSize); -#endif - -#ifdef MODEL_V3 - ctx->inferenceNetwork = new YoloV3(batchSize); -#endif - - return ctx; -} - -std::vector YoloPluginProcess(YoloPluginCtx* ctx, std::vector& cvmats) -{ - std::vector outputs = std::vector(cvmats.size(), nullptr); - cv::Mat preprocessedImages; - struct timeval preStart, preEnd, inferStart, inferEnd, postStart, postEnd; - double preElapsed = 0.0, inferElapsed = 0.0, postElapsed = 0.0; - - if (cvmats.size() > 0) - { - gettimeofday(&preStart, NULL); - dsPreProcessBatchInput(cvmats, preprocessedImages, ctx->initParams.processingWidth, - ctx->initParams.processingHeight, ctx->inferenceNetwork->getInputH(), - ctx->inferenceNetwork->getInputW()); - gettimeofday(&preEnd, NULL); - - gettimeofday(&inferStart, NULL); - ctx->inferenceNetwork->doInference(preprocessedImages.data); - gettimeofday(&inferEnd, NULL); - - gettimeofday(&postStart, NULL); - decodeBatchDetections(ctx, outputs); - gettimeofday(&postEnd, NULL); - } - - // Perf calc - if (ctx->inferenceNetwork->isPrintPerfInfo()) - { - preElapsed - = ((preEnd.tv_sec - preStart.tv_sec) + (preEnd.tv_usec - preStart.tv_usec) / 1000000.0) - * (1000 / ctx->batchSize); - inferElapsed = ((inferEnd.tv_sec - inferStart.tv_sec) - + (inferEnd.tv_usec - inferStart.tv_usec) / 1000000.0) - * (1000 / ctx->batchSize); - postElapsed = ((postEnd.tv_sec - postStart.tv_sec) - + (postEnd.tv_usec - postStart.tv_usec) / 1000000.0) - * (1000 / ctx->batchSize); - - ctx->inferTime += inferElapsed; - ctx->preTime += preElapsed; - ctx->postTime += postElapsed; - ++ctx->batchCount; - } - return outputs; -} - -void YoloPluginCtxDeinit(YoloPluginCtx* ctx) -{ - if (ctx->inferenceNetwork->isPrintPerfInfo()) - { - std::cout << "DS Example Perf Summary " << std::endl; - std::cout << "Batch Size : " << ctx->batchSize << std::endl; - std::cout << std::fixed << std::setprecision(4) - << "PreProcess : " << ctx->preTime / ctx->batchCount - << " ms Inference : " << ctx->inferTime / ctx->batchCount - << " ms PostProcess : " << ctx->postTime / ctx->batchCount << " ms Total : " - << (ctx->preTime + ctx->postTime + ctx->inferTime) / ctx->batchCount - << " ms per Image" << std::endl; - } - - delete ctx->inferenceNetwork; - delete ctx; -} diff --git a/sources/gst-yoloplugin/yoloplugin_lib/yoloplugin_lib.h b/sources/gst-yoloplugin/yoloplugin_lib/yoloplugin_lib.h deleted file mode 100644 index 31f70ce..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/yoloplugin_lib.h +++ /dev/null @@ -1,92 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ -#ifndef __YOLOPLUGIN_LIB__ -#define __YOLOPLUGIN_LIB__ - -#include - -#include "calibrator.h" -#include "trt_utils.h" -#include "yolo.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define MAX_OBJECTS_PER_FRAME 32 -typedef struct YoloPluginCtx YoloPluginCtx; -typedef struct YoloPluginOutput YoloPluginOutput; -// Init parameters structure as input, required for instantiating yoloplugin_lib -typedef struct -{ - // Width at which frame/object will be scaled - int processingWidth; - // height at which frame/object will be scaled - int processingHeight; - // Flag to indicate whether operating on crops of full frame - int fullFrame; -} YoloPluginInitParams; - -struct YoloPluginCtx -{ - YoloPluginInitParams initParams; - Yolo* inferenceNetwork; - - // perf vars - float inferTime = 0.0, preTime = 0.0, postTime = 0.0; - int batchCount = 0, batchSize = 0; -}; - -// Detected/Labelled object structure, stores bounding box info along with label -typedef struct -{ - int left; - int top; - int width; - int height; - char label[64]; -} YoloPluginObject; - -// Output data returned after processing -struct YoloPluginOutput -{ - int numObjects; - YoloPluginObject object[MAX_OBJECTS_PER_FRAME]; -}; - -// Initialize library context -YoloPluginCtx* YoloPluginCtxInit(YoloPluginInitParams* initParams, size_t batchSize); - -// Dequeue processed output -std::vector YoloPluginProcess(YoloPluginCtx* ctx, std::vector& cvmats); - -// Deinitialize library context -void YoloPluginCtxDeinit(YoloPluginCtx* ctx); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/sources/gst-yoloplugin/yoloplugin_lib/yolov2.cpp b/sources/gst-yoloplugin/yoloplugin_lib/yolov2.cpp deleted file mode 100644 index f5e56a5..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/yolov2.cpp +++ /dev/null @@ -1,141 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#include "yolov2.h" -#include "network_config.h" - -YoloV2::YoloV2(uint batchSize) : - Yolo(batchSize), - m_Stride(config::kSTRIDE), - m_GridSize(config::kGRID_SIZE), - m_OutputIndex(-1), - m_OutputSize(config::kOUTPUT_SIZE), - m_OutputBlobName(config::kOUTPUT_BLOB_NAME) -{ - assert(m_NetworkType == "yolov2"); - // Allocate Buffers - m_OutputIndex = m_Engine->getBindingIndex(m_OutputBlobName.c_str()); - assert(m_OutputIndex != -1); - NV_CUDA_CHECK( - cudaMalloc(&m_Bindings.at(m_InputIndex), m_BatchSize * m_InputSize * sizeof(float))); - NV_CUDA_CHECK( - cudaMalloc(&m_Bindings.at(m_OutputIndex), m_BatchSize * m_OutputSize * sizeof(float))); - m_TrtOutputBuffers.front() = new float[m_OutputSize * m_BatchSize]; -}; - -void YoloV2::doInference(const unsigned char* input) -{ - NV_CUDA_CHECK(cudaMemcpyAsync(m_Bindings.at(m_InputIndex), input, - m_BatchSize * m_InputSize * sizeof(float), cudaMemcpyHostToDevice, - m_CudaStream)); - - m_Context->enqueue(m_BatchSize, m_Bindings.data(), m_CudaStream, nullptr); - - NV_CUDA_CHECK(cudaMemcpyAsync(m_TrtOutputBuffers.at(0), m_Bindings.at(m_OutputIndex), - m_BatchSize * m_OutputSize * sizeof(float), - cudaMemcpyDeviceToHost, m_CudaStream)); - cudaStreamSynchronize(m_CudaStream); -} - -std::vector YoloV2::decodeDetections(const int& imageIdx, const int& imageH, - const int& imageW) -{ - std::vector binfo; - const float* detections = &m_TrtOutputBuffers.at(0)[imageIdx * m_OutputSize]; - float scalingFactor - = std::min(static_cast(m_InputW) / imageW, static_cast(m_InputH) / imageH); - for (uint y = 0; y < m_GridSize; y++) - { - for (uint x = 0; x < m_GridSize; x++) - { - for (uint b = 0; b < m_NumBBoxes; b++) - { - const float pw = m_Anchors[2 * b]; - const float ph = m_Anchors[2 * b + 1]; - const int numGridCells = m_GridSize * m_GridSize; - const int bbindex = y * m_GridSize + x; - const float bx - = x + detections[bbindex + numGridCells * (b * (5 + m_NumOutputClasses) + 0)]; - const float by - = y + detections[bbindex + numGridCells * (b * (5 + m_NumOutputClasses) + 1)]; - const float bw = pw - * exp(detections[bbindex + numGridCells * (b * (5 + m_NumOutputClasses) + 2)]); - const float bh = ph - * exp(detections[bbindex + numGridCells * (b * (5 + m_NumOutputClasses) + 3)]); - - const float objectness - = detections[bbindex + numGridCells * (b * (5 + m_NumOutputClasses) + 4)]; - float maxProb = 0.0f; - int maxIndex = -1; - - for (uint i = 0; i < m_NumOutputClasses; i++) - { - float prob - = detections[bbindex - + numGridCells * (b * (5 + m_NumOutputClasses) + (5 + i))]; - - if (prob > maxProb) - { - maxProb = prob; - maxIndex = i; - } - } - - maxProb = objectness * maxProb; - - if (maxProb > m_ProbThresh) - { - BBoxInfo bbi; - bbi.box = convertBBox(bx, by, bw, bh, m_Stride); - - // Undo Letterbox - float xCorrection = (m_InputW - scalingFactor * imageW) / 2; - float yCorrection = (m_InputH - scalingFactor * imageH) / 2; - bbi.box.x1 -= xCorrection; - bbi.box.x2 -= xCorrection; - bbi.box.y1 -= yCorrection; - bbi.box.y2 -= yCorrection; - - // Restore to input image resolution - bbi.box.x1 /= scalingFactor; - bbi.box.x2 /= scalingFactor; - bbi.box.y1 /= scalingFactor; - bbi.box.y2 /= scalingFactor; - - bbi.box.x1 = clamp(bbi.box.x1, 0, imageW); - bbi.box.x2 = clamp(bbi.box.x2, 0, imageW); - bbi.box.y1 = clamp(bbi.box.y1, 0, imageH); - bbi.box.y2 = clamp(bbi.box.y2, 0, imageH); - - bbi.label = maxIndex; - bbi.prob = maxProb; - - binfo.push_back(bbi); - } - } - } - } - return binfo; -} diff --git a/sources/gst-yoloplugin/yoloplugin_lib/yolov2.h b/sources/gst-yoloplugin/yoloplugin_lib/yolov2.h deleted file mode 100644 index 50d1ca3..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/yolov2.h +++ /dev/null @@ -1,51 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#ifndef _YOLO_V2_ -#define _YOLO_V2_ - -#include "yolo.h" - -#include -#include -#include - -class YoloV2 : public Yolo -{ -public: - explicit YoloV2(const uint batchSize); - void doInference(const unsigned char* input) override; - std::vector decodeDetections(const int& imageIdx, const int& imageH, - const int& imageW) override; - -private: - const uint m_Stride; - const uint m_GridSize; - int m_OutputIndex; - const uint64_t m_OutputSize; - const std::string m_OutputBlobName; -}; - -#endif // _YOLO_V2_ \ No newline at end of file diff --git a/sources/gst-yoloplugin/yoloplugin_lib/yolov3.cpp b/sources/gst-yoloplugin/yoloplugin_lib/yolov3.cpp deleted file mode 100644 index 0e48952..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/yolov3.cpp +++ /dev/null @@ -1,197 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ -#include "yolov3.h" -#include "network_config.h" - -YoloV3::YoloV3(uint batchSize) : - Yolo(batchSize), - m_Stride1(config::kSTRIDE_1), - m_Stride2(config::kSTRIDE_2), - m_Stride3(config::kSTRIDE_3), - m_GridSize1(config::kGRID_SIZE_1), - m_GridSize2(config::kGRID_SIZE_2), - m_GridSize3(config::kGRID_SIZE_3), - m_OutputIndex1(-1), - m_OutputIndex2(-1), - m_OutputIndex3(-1), - m_OutputSize1(config::kOUTPUT_SIZE_1), - m_OutputSize2(config::kOUTPUT_SIZE_2), - m_OutputSize3(config::kOUTPUT_SIZE_3), - m_Mask1(config::kMASK_1), - m_Mask2(config::kMASK_2), - m_Mask3(config::kMASK_3), - m_OutputBlobName1(config::kOUTPUT_BLOB_NAME_1), - m_OutputBlobName2(config::kOUTPUT_BLOB_NAME_2), - m_OutputBlobName3(config::kOUTPUT_BLOB_NAME_3) -{ - assert(m_NetworkType == "yolov3"); - // Allocate Buffers - m_OutputIndex1 = m_Engine->getBindingIndex(m_OutputBlobName1.c_str()); - assert(m_OutputIndex1 != -1); - m_OutputIndex2 = m_Engine->getBindingIndex(m_OutputBlobName2.c_str()); - assert(m_OutputIndex2 != -1); - m_OutputIndex3 = m_Engine->getBindingIndex(m_OutputBlobName3.c_str()); - assert(m_OutputIndex3 != -1); - NV_CUDA_CHECK( - cudaMalloc(&m_Bindings.at(m_InputIndex), m_BatchSize * m_InputSize * sizeof(float))); - NV_CUDA_CHECK( - cudaMalloc(&m_Bindings.at(m_OutputIndex1), m_BatchSize * m_OutputSize1 * sizeof(float))); - NV_CUDA_CHECK( - cudaMalloc(&m_Bindings.at(m_OutputIndex2), m_BatchSize * m_OutputSize2 * sizeof(float))); - NV_CUDA_CHECK( - cudaMalloc(&m_Bindings.at(m_OutputIndex3), m_BatchSize * m_OutputSize3 * sizeof(float))); - m_TrtOutputBuffers.at(0) = new float[m_OutputSize1 * m_BatchSize]; - m_TrtOutputBuffers.at(1) = new float[m_OutputSize2 * m_BatchSize]; - m_TrtOutputBuffers.at(2) = new float[m_OutputSize3 * m_BatchSize]; -}; - -void YoloV3::doInference(const unsigned char* input) -{ - NV_CUDA_CHECK(cudaMemcpyAsync(m_Bindings.at(m_InputIndex), input, - m_BatchSize * m_InputSize * sizeof(float), cudaMemcpyHostToDevice, - m_CudaStream)); - - m_Context->enqueue(m_BatchSize, m_Bindings.data(), m_CudaStream, nullptr); - - NV_CUDA_CHECK(cudaMemcpyAsync(m_TrtOutputBuffers.at(0), m_Bindings.at(m_OutputIndex1), - m_BatchSize * m_OutputSize1 * sizeof(float), - cudaMemcpyDeviceToHost, m_CudaStream)); - NV_CUDA_CHECK(cudaMemcpyAsync(m_TrtOutputBuffers.at(1), m_Bindings.at(m_OutputIndex2), - m_BatchSize * m_OutputSize1 * sizeof(float), - cudaMemcpyDeviceToHost, m_CudaStream)); - NV_CUDA_CHECK(cudaMemcpyAsync(m_TrtOutputBuffers.at(2), m_Bindings.at(m_OutputIndex3), - m_BatchSize * m_OutputSize1 * sizeof(float), - cudaMemcpyDeviceToHost, m_CudaStream)); - - cudaStreamSynchronize(m_CudaStream); -} - -std::vector YoloV3::decodeDetections(const int& imageIdx, const int& imageH, - const int& imageW) -{ - std::vector binfo; - - std::vector binfo1 - = decodeTensor(imageH, imageW, &m_TrtOutputBuffers.at(0)[imageIdx * m_OutputSize1], m_Mask1, - m_GridSize1, m_Stride1); - std::vector binfo2 - = decodeTensor(imageH, imageW, &m_TrtOutputBuffers.at(1)[imageIdx * m_OutputSize2], m_Mask2, - m_GridSize2, m_Stride2); - std::vector binfo3 - = decodeTensor(imageH, imageW, &m_TrtOutputBuffers.at(2)[imageIdx * m_OutputSize3], m_Mask3, - m_GridSize3, m_Stride3); - - binfo.insert(binfo.end(), binfo1.begin(), binfo1.end()); - binfo.insert(binfo.end(), binfo2.begin(), binfo2.end()); - binfo.insert(binfo.end(), binfo3.begin(), binfo3.end()); - - return binfo; -} - -std::vector YoloV3::decodeTensor(const int& imageH, const int& imageW, - const float* detections, const std::vector mask, - const uint gridSize, const uint stride) -{ - float scalingFactor - = std::min(static_cast(m_InputW) / imageW, static_cast(m_InputH) / imageH); - std::vector binfo; - for (uint y = 0; y < gridSize; ++y) - { - for (uint x = 0; x < gridSize; ++x) - { - for (uint b = 0; b < m_NumBBoxes; ++b) - { - const float pw = m_Anchors[mask[b] * 2]; - const float ph = m_Anchors[mask[b] * 2 + 1]; - - const int numGridCells = gridSize * gridSize; - const int bbindex = y * gridSize + x; - const float bx - = x - + sigmoid( - detections[bbindex + numGridCells * (b * (5 + m_NumOutputClasses) + 0)]); - const float by - = y - + sigmoid( - detections[bbindex + numGridCells * (b * (5 + m_NumOutputClasses) + 1)]); - const float bw = pw - * exp(detections[bbindex + numGridCells * (b * (5 + m_NumOutputClasses) + 2)]); - const float bh = ph - * exp(detections[bbindex + numGridCells * (b * (5 + m_NumOutputClasses) + 3)]); - - const float objectness = sigmoid( - detections[bbindex + numGridCells * (b * (5 + m_NumOutputClasses) + 4)]); - - float maxProb = 0.0f; - int maxIndex = -1; - - for (uint i = 0; i < m_NumOutputClasses; ++i) - { - float prob = sigmoid( - detections[bbindex - + numGridCells * (b * (5 + m_NumOutputClasses) + (5 + i))]); - - if (prob > maxProb) - { - maxProb = prob; - maxIndex = i; - } - } - maxProb = objectness * maxProb; - - if (maxProb > m_ProbThresh) - { - BBoxInfo bbi; - bbi.box = convertBBox(bx, by, bw, bh, stride); - - // Undo Letterbox - float x_correction = (m_InputW - scalingFactor * imageW) / 2; - float y_correction = (m_InputH - scalingFactor * imageH) / 2; - bbi.box.x1 -= x_correction; - bbi.box.x2 -= x_correction; - bbi.box.y1 -= y_correction; - bbi.box.y2 -= y_correction; - - // Restore to input resolution - bbi.box.x1 /= scalingFactor; - bbi.box.x2 /= scalingFactor; - bbi.box.y1 /= scalingFactor; - bbi.box.y2 /= scalingFactor; - - bbi.box.x1 = clamp(bbi.box.x1, 0, imageW); - bbi.box.x2 = clamp(bbi.box.x2, 0, imageW); - bbi.box.y1 = clamp(bbi.box.y1, 0, imageH); - bbi.box.y2 = clamp(bbi.box.y2, 0, imageH); - - bbi.label = maxIndex; - bbi.prob = maxProb; - - binfo.push_back(bbi); - } - } - } - } - return binfo; -} diff --git a/sources/gst-yoloplugin/yoloplugin_lib/yolov3.h b/sources/gst-yoloplugin/yoloplugin_lib/yolov3.h deleted file mode 100644 index 25e65e5..0000000 --- a/sources/gst-yoloplugin/yoloplugin_lib/yolov3.h +++ /dev/null @@ -1,67 +0,0 @@ -/** -MIT License - -Copyright (c) 2018 NVIDIA CORPORATION. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -* -*/ - -#ifndef _YOLO_V3_ -#define _YOLO_V3_ - -#include "yolo.h" - -#include -#include -#include - -class YoloV3 : public Yolo -{ -public: - explicit YoloV3(const uint batchSize); - void doInference(const unsigned char* input) override; - std::vector decodeDetections(const int& imageIdx, const int& imageH, - const int& imageW) override; - -private: - std::vector decodeTensor(const int& imageH, const int& imageW, - const float* dectections, std::vector mask, - const uint gridSize, const uint stride); - const uint m_Stride1; - const uint m_Stride2; - const uint m_Stride3; - const uint m_GridSize1; - const uint m_GridSize2; - const uint m_GridSize3; - int m_OutputIndex1; - int m_OutputIndex2; - int m_OutputIndex3; - const uint64_t m_OutputSize1; - const uint64_t m_OutputSize2; - const uint64_t m_OutputSize3; - const std::vector m_Mask1; - const std::vector m_Mask2; - const std::vector m_Mask3; - const std::string m_OutputBlobName1; - const std::string m_OutputBlobName2; - const std::string m_OutputBlobName3; -}; - -#endif // _YOLO_V3_ \ No newline at end of file