demos: Add first version of HexagonTFLite demosHEADmaster

Signed-off-by: Srinivas Kandagatla <srinivas.kandagatla@linaro.org>

3 files changed, 481 insertions, 0 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 0000000..f8c4ff5
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,59 @@
+cmake_minimum_required(VERSION 3.20)
+
+project( HexagonTFLiteDemos )
+find_package( OpenCV REQUIRED )
+set(CMAKE_SYSTEM_NAME Linux)
+set(CMAKE_SYSTEM_PROCESSOR aarch64)
+set(CMAKE_C_COMPILER aarch64-linux-gnu-gcc)
+set(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++)
+set(GCC_COMPILER_VERSION "" CACHE STRING "GCC Compiler version")
+set(GNU_MACHINE "aarch64-linux-gnu" CACHE STRING "GNU compiler triple")
+
+
+if (NOT OPENCV_CROSS_INSTALL_DIR)
+	message(FATAL_ERROR
+		"Please specify path to OpenCV install directory using OPENCV_CROSS_INSTALL_DIR in cmdline.\n")
+endif()
+
+if (NOT TFLITE_BUILD_DIR)
+	message(FATAL_ERROR
+		"Please specify path to TensorFlowLite build directory using TFLITE_BUILD_DIR in cmdline.\n")
+endif()
+
+if (NOT TFLITE_SRC_DIR)
+	message(FATAL_ERROR
+		"Please specify path to TensorFlowLite build directory using TFLITE_SRC_DIR in cmdline.\n")
+endif()
+
+include_directories(${OPENCV_CROSS_INSTALL_DIR}/include/opencv4/  ${TFLITE_SRC_DIR} )
+include_directories( ${TFLITE_BUILD_DIR}/bin/external/flatbuffers/_virtual_includes/flatbuffers/ )
+add_executable(MobileNetSSD mobilenetssd.cpp )
+add_executable(MobileNet mobilenet.cpp )
+link_directories(${OPENCV_CROSS_INSTALL_DIR}/lib)
+link_directories(${TFLITE_BUILD_DIR}/bin/tensorflow/lite/)
+set(CMAKE_LIBRARY_PATH ${TFLITE_BUILD_DIR}/bin/tensorflow/lite)
+target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libhexagon_delegate.a)
+target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libhexagon_implementation.a)
+target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libhexagon_delegate_kernel.a)
+target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libutils.a)
+target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/builders/libop_builder.a)
+target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/libserialization.a)
+target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/utils/libsimple_delegate.a)
+target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/tools/evaluation/libutils.a)
+target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/libutils.a)
+target_link_libraries( MobileNetSSD ${TFLITE_BUILD_DIR}/bin/external/farmhash_archive/libfarmhash.a)
+
+target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libhexagon_delegate.a)
+target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libhexagon_implementation.a)
+target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libhexagon_delegate_kernel.a)
+target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/libutils.a)
+target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/hexagon/builders/libop_builder.a)
+target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/libserialization.a)
+target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/utils/libsimple_delegate.a)
+target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/tools/evaluation/libutils.a)
+target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/tensorflow/lite/delegates/libutils.a)
+target_link_libraries( MobileNet ${TFLITE_BUILD_DIR}/bin/external/farmhash_archive/libfarmhash.a)
+
+
+target_link_libraries( MobileNetSSD  -ldl -lstdc++ -lm -ltensorflowlite -L${TFLITE_BUILD_DIR}/bin/tensorflow/lite -lopencv_dnn -lopencv_highgui -lopencv_photo -lopencv_stitching -lopencv_core -lopencv_video -lopencv_videoio  -ldl -lopencv_imgcodecs -lopencv_calib3d -lopencv_features2d -lopencv_flann -lopencv_imgproc -lopencv_core -L${OPENCV_CROSS_INSTALL_DIR}/lib)
+target_link_libraries( MobileNet  -ldl -lstdc++ -lm -ltensorflowlite -L${TFLITE_BUILD_DIR}/bin/tensorflow/lite -lopencv_dnn -lopencv_highgui -lopencv_photo -lopencv_stitching -lopencv_core -lopencv_video -lopencv_videoio  -ldl -lopencv_imgcodecs -lopencv_calib3d -lopencv_features2d -lopencv_flann -lopencv_imgproc -lopencv_core -L${OPENCV_CROSS_INSTALL_DIR}/lib)
diff --git a/mobilenet.cpp b/mobilenet.cpp
new file mode 100644
index 0000000..621b09e
--- /dev/null
+++ b/mobilenet.cpp
@@ -0,0 +1,212 @@
+#include <iostream>
+#include <fstream>
+#include <cstdlib>
+#include <vector>
+#include <string>
+#include <iterator>
+#include <unordered_map>
+#include <algorithm>
+#include <algorithm>
+#include <functional>
+#include <queue>
+#include <stdlib.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <math.h>
+#include <stdint.h>
+#include <assert.h>
+#include <dirent.h> 
+
+#include <opencv2/core.hpp>
+#include <opencv2/videoio.hpp>
+#include <opencv2/highgui.hpp>
+#include <opencv2/imgproc.hpp>
+
+#include "tensorflow/lite/kernels/register.h"
+#include <tensorflow/lite/model.h>
+#include <tensorflow/lite/interpreter.h>
+#include <tensorflow/lite/delegates/hexagon/hexagon_delegate.h>
+
+#include <tensorflow/lite/kernels/register.h>
+#include <tensorflow/lite/optional_debug_tools.h>
+#include <tensorflow/lite/profiling/profiler.h>
+#include <tensorflow/lite/string_util.h>
+#include <tensorflow/lite/tools/command_line_flags.h>
+#include <tensorflow/lite/tools/delegates/delegate_provider.h>
+#include <tensorflow/lite/builtin_op_data.h>
+
+using namespace cv;
+using std::cout; using std::cerr; using std::endl;
+
+#define IMG_HEIGHT		224
+#define IMG_WIDTH		224
+#define IMG_DEPTH		3
+#define IMG_ELEMENT_SIZE	1
+#define BUFFER_SIZE	(IMG_ELEMENT_SIZE * IMG_WIDTH * IMG_HEIGHT * IMG_DEPTH)
+#define NUM_CATEGORIES	1001
+#define IMG_DIR "/home/linaro/apps/256_ObjectCategories-jpegs/"
+#define NUM_FRAMES	16
+#define MODEL_NAME "/home/linaro/apps/mobilenet_quant_v1_224.tflite"
+#define LABELS_FILE "/home/linaro/apps/labels.txt"
+
+std::vector<std::string> labels;
+
+int read_labels_from_file(void)
+{
+        std::ifstream ifs(LABELS_FILE);
+	if (!ifs.is_open()) {
+		cout << "File " << LABELS_FILE << " not found" << endl;
+		return -1;
+	}
+	std::string line;
+
+	while (std::getline(ifs, line))
+		labels.push_back(line);
+
+	return 0;
+}
+
+struct indexed_data {
+        float f;
+        int i;
+};
+
+int indexed_data_sort(const void *va, const void *vb)
+{
+        const struct indexed_data *a = (const struct indexed_data *)va;
+        const struct indexed_data *b = (const struct indexed_data *)vb;
+
+        if (a->f > b->f) return -1;
+        if (a->f < b->f) return 1;
+        return 0;
+}
+
+struct indexed_data data[NUM_CATEGORIES];
+
+static float get_top5_match(uint8_t *values, int n_entries,
+			    Mat *frame)
+{
+	const float threshold = 0.001f;
+        int i, k;
+	char label[256];
+	int fontface = cv::FONT_HERSHEY_PLAIN;
+	double scale = 1.0;
+	int thickness = 1;
+	int baseline = 0;
+	float fvalue;
+	int count = 0;
+
+	for (i = 0; i < NUM_CATEGORIES; i++) {
+		float fvalue = values[i] / 255.0;
+		if  (fvalue > threshold) {
+	                data[count].f = fvalue;
+		        data[count].i = i;
+			count++;
+		}
+	}
+	qsort(data, count, sizeof(struct indexed_data), indexed_data_sort);
+        for (i = 0; i < 5; i++) {
+		k = (i + 1)*20;
+                sprintf(label, "%02.2f%%, %s",
+			data[i].f * 100,  labels[data[i].i].c_str());
+		cv::Size text = cv::getTextSize(label, fontface, scale,
+						thickness, &baseline);
+		cv::rectangle(*frame, cv::Point(30, k) + cv::Point(0, baseline),
+			      cv::Point(30, k) + cv::Point(text.width, -text.height),
+			      CV_RGB(0,0,0), cv::FILLED);
+		cv::putText(*frame, label, cv::Point(30,k), cv::FONT_HERSHEY_PLAIN,
+			    1.0, cv::Scalar(57,255,20), 1, LINE_8);
+
+        }
+	return 0;
+}
+
+int main(int, char**)
+{
+	std::unique_ptr<tflite::FlatBufferModel> model;
+	tflite::ops::builtin::BuiltinOpResolver resolver;
+	std::unique_ptr<tflite::Interpreter> interpreter;
+	TfLiteHexagonDelegateOptions params = {0};
+	const char* path = "/dsp/cdsp/";
+	uint8_t *input_8, *output_8;
+	struct dirent *dentry;
+	Mat frame, rsz_frame;
+	size_t label_count;
+	char img_path[1024];
+	int ret, idx = 0;
+	char fname[16];
+	DIR *d;
+
+	model = tflite::FlatBufferModel::BuildFromFile(MODEL_NAME);
+	if (!model) {
+		std::cout << "Failed to mmap model ";
+		exit(-1);
+	}
+	tflite::InterpreterBuilder(*model, resolver)(&interpreter);
+	if (!interpreter) {
+		std::cout << "Failed to construct interpreter";
+		exit(-1);
+	}
+	interpreter->SetAllowFp16PrecisionForFp32(false);
+	interpreter->SetNumThreads(4);
+
+	std::cout << "Enabling Hexagon Delegate!\n";
+
+	TfLiteHexagonInitWithPath(path);
+	auto *delegate = TfLiteHexagonDelegateCreate(&params);
+
+	interpreter->ModifyGraphWithDelegate(delegate);
+
+	if (interpreter->AllocateTensors() != kTfLiteOk) {
+		std::cout << "Failed to allocate tensors!" << endl;
+		exit(-1);
+	}
+
+	const std::vector<int> inputs = interpreter->inputs();
+	int input = interpreter->inputs()[0];
+	int output = interpreter->outputs()[0];
+
+	TfLiteIntArray* output_dims = interpreter->tensor(output)->dims;
+	auto output_size = output_dims->data[output_dims->size - 1];
+	TfLiteType input_type = interpreter->tensor(input)->type;
+
+	/* We ONLY support UInt8 on this quantized model */
+	input_8 = interpreter->typed_tensor<uint8_t>(input);
+	output_8 = interpreter->typed_output_tensor<uint8_t>(0);
+
+	Mat rgb_mat(IMG_HEIGHT, IMG_WIDTH, CV_8UC3, input_8);
+
+	if (read_labels_from_file())
+		exit(-1);
+
+	d = opendir(IMG_DIR);
+	if (!d) {
+		printf("Error opening %s directory\n", IMG_DIR);
+		return -1;
+	}
+
+	while ((dentry = readdir(d)) != NULL) {
+		if (!strcmp(dentry->d_name, ".") || !strcmp(dentry->d_name, ".."))
+			 continue;
+		idx = idx%NUM_FRAMES;
+		sprintf(fname, "Frame%d\n", idx++);
+		sprintf(img_path, "%s%s",IMG_DIR, dentry->d_name);
+		frame = imread(img_path, 1);
+
+		cv::resize(frame, frame, cv::Size(IMG_HEIGHT, IMG_WIDTH));
+		cv::cvtColor(frame, rgb_mat, cv::COLOR_BGR2RGB);
+
+		if (interpreter->Invoke() != kTfLiteOk) {
+			std::cout << "Failed to invoke tflite!";
+			exit(-1);
+		}
+
+		get_top5_match(output_8, output_size, &rgb_mat);
+		imshow(fname, rgb_mat);
+
+		waitKey(1);
+	}
+
+	closedir(d);
+	return 0;
+}
diff --git a/mobilenetssd.cpp b/mobilenetssd.cpp
new file mode 100644
index 0000000..2728f84
--- /dev/null
+++ b/mobilenetssd.cpp
@@ -0,0 +1,210 @@
+#include <iostream>
+#include <fstream>
+#include <cstdlib>
+#include <vector>
+#include <string>
+#include <iterator>
+#include <unordered_map>
+#include <algorithm>
+#include <algorithm>
+#include <functional>
+#include <queue>
+#include <stdlib.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <math.h>
+#include <stdint.h>
+#include <assert.h>
+#include <dirent.h>
+
+#include <opencv2/core.hpp>
+#include <opencv2/videoio.hpp>
+#include <opencv2/highgui.hpp>
+#include <opencv2/imgproc.hpp>
+
+#include <tensorflow/lite/kernels/register.h>
+#include <tensorflow/lite/model.h>
+#include <tensorflow/lite/interpreter.h>
+#include <tensorflow/lite/delegates/hexagon/hexagon_delegate.h>
+
+#include <tensorflow/lite/kernels/register.h>
+#include <tensorflow/lite/optional_debug_tools.h>
+#include <tensorflow/lite/profiling/profiler.h>
+#include <tensorflow/lite/string_util.h>
+#include <tensorflow/lite/tools/command_line_flags.h>
+#include <tensorflow/lite/tools/delegates/delegate_provider.h>
+#include <tensorflow/lite/builtin_op_data.h>
+
+using namespace cv;
+using std::cout; using std::cerr; using std::endl;
+
+#define IMG_HEIGHT		300
+#define IMG_WIDTH		300
+#define IMG_DEPTH		3
+#define IMG_ELEMENT_SIZE	1
+#define BUFFER_SIZE	(IMG_ELEMENT_SIZE * IMG_WIDTH * IMG_HEIGHT * IMG_DEPTH)
+#define MODEL_NAME "/home/linaro/apps/coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.tflite"
+#define LABELS_FILE "/home/linaro/apps/labelmap.txt"
+
+std::vector<std::string> labels;
+
+int read_labels_from_file(void)
+{
+        std::ifstream ifs(LABELS_FILE);
+	if (!ifs.is_open()) {
+		cout << "File " << LABELS_FILE << " not found" << endl;
+		return -1;
+	}
+	std::string line;
+
+	while (std::getline(ifs, line))
+		labels.push_back(line);
+
+	return 0;
+}
+
+/* Max detection Items */
+#define MAX_ITEMS		10
+
+/* Minimum Accuracy required in percentage */
+#define MIN_ACCURACY		40
+
+struct SSDbox {
+	float top;
+	float left;
+	float bottom;
+	float right;
+	uint32_t id;
+	float score;
+};
+static struct SSDbox SSDBoxes[MAX_ITEMS];
+
+void SSDshowOutput(float *scores, float *mclasses, float *boxes, Mat *frame)
+{
+	int fontface = cv::FONT_HERSHEY_PLAIN;
+	double scale = 1.0;
+	int thickness = 1;
+	char label[256];
+	int baseline = 0;
+	int i;
+
+	for ( i = 0; i < MAX_ITEMS; i++) {
+		SSDBoxes[i].top = frame->rows *  boxes[i * 4];
+		SSDBoxes[i].left = frame->cols *  boxes[i * 4 + 1];
+		SSDBoxes[i].bottom = frame->rows * boxes[i * 4 + 2];
+		SSDBoxes[i].right = frame->cols * boxes[i * 4 + 3];
+		SSDBoxes[i].id = (uint32_t)mclasses[i] + 1;
+		SSDBoxes[i].score = scores[i] * 100.00;
+	}
+
+	for ( i = 0; i < MAX_ITEMS; i++) {
+		if (SSDBoxes[i].score < MIN_ACCURACY)
+			break;
+#if DEBUG
+		std::cout << i <<":\t" << SSDBoxes[i].score << ":\t" <<
+			labels[(SSDBoxes[i].id) ] <<":\t(" <<
+			(uint32_t) SSDBoxes[i].top << ", " <<
+			(uint32_t)SSDBoxes[i].left << ", " <<
+			(uint32_t) SSDBoxes[i].bottom << ", " <<
+			(uint32_t) SSDBoxes[i].right << ")"<< std::endl;
+#endif
+		sprintf(label, "%02.2f%%, %s", SSDBoxes[i].score,
+			labels[SSDBoxes[i].id].c_str());
+		cv::rectangle(*frame, cv::Point((uint32_t) SSDBoxes[i].left, (uint32_t)SSDBoxes[i].top),
+			      cv::Point( (uint32_t) SSDBoxes[i].right, (uint32_t) SSDBoxes[i].bottom),
+			      cv::Scalar(0,255, 0), 1);
+
+		cv::Size text = cv::getTextSize(label, fontface, scale,	thickness, &baseline);
+		cv::rectangle(*frame, cv::Point((uint32_t) SSDBoxes[i].left, (uint32_t)SSDBoxes[i].top) +
+			      cv::Point(0, baseline), cv::Point((uint32_t) SSDBoxes[i].left, (uint32_t)SSDBoxes[i].top) +
+			      cv::Point(text.width, -text.height), CV_RGB(0,0,0), cv::FILLED);
+		cv::putText(*frame, label, cv::Point((uint32_t) SSDBoxes[i].left, (uint32_t)SSDBoxes[i].top),
+			    cv::FONT_HERSHEY_PLAIN, 1.0, cv::Scalar(57,255,20), 0.1, LINE_8);
+	}
+}
+
+int main(int, char**)
+{
+	std::unique_ptr<tflite::FlatBufferModel> model;
+	tflite::ops::builtin::BuiltinOpResolver resolver;
+	std::unique_ptr<tflite::Interpreter> interpreter;
+	TfLiteHexagonDelegateOptions params = {0};
+	const char* path = "/dsp/cdsp/";
+	Mat frame, frame1;
+	size_t label_count;
+	char img_path[1024];
+	int ret, idx = 0;
+	char fname[16];
+	DIR *d;
+
+	model = tflite::FlatBufferModel::BuildFromFile(MODEL_NAME);
+	if (!model) {
+		std::cout << "Failed to mmap model ";
+		exit(-1);
+	}
+	tflite::InterpreterBuilder(*model, resolver)(&interpreter);
+	if (!interpreter) {
+		std::cout << "Failed to construct interpreter";
+		exit(-1);
+	}
+	interpreter->SetAllowFp16PrecisionForFp32(false);
+	interpreter->SetNumThreads(4);
+
+	std::cout << "Enabling Hexagon Delegate!\n";
+
+	TfLiteHexagonInitWithPath(path);
+	auto *delegate = TfLiteHexagonDelegateCreate(&params);
+
+	interpreter->ModifyGraphWithDelegate(delegate);
+
+	if (interpreter->AllocateTensors() != kTfLiteOk) {
+		std::cout << "Failed to allocate tensors!" << endl;
+		exit(-1);
+	}
+
+	const std::vector<int> inputs = interpreter->inputs();
+	const std::vector<int> outputs = interpreter->outputs();
+	int input = interpreter->inputs()[0];
+	/* We ONLY support UInt8 on this quantized model */
+	uint8_t *finput = interpreter->typed_tensor<uint8_t>(input);
+	float *locations, *classes, *scores, *dcount;
+
+	locations = interpreter->typed_output_tensor<float>(0);
+	classes = interpreter->typed_output_tensor<float>(1);
+	scores = interpreter->typed_output_tensor<float>(2);
+	dcount = interpreter->typed_output_tensor<float>(3);
+
+	if (read_labels_from_file())
+		exit(-1);
+
+	Mat rgb_mat(IMG_HEIGHT, IMG_WIDTH, CV_8UC3, finput);
+	VideoCapture capture(0);
+	if (!capture.isOpened())
+	{
+		cerr << "ERROR: Can't initialize camera capture" << endl;
+		return 1;
+	}
+
+	for (;;)
+	{
+		capture >> frame;
+		if (frame.empty())
+		{
+			cerr << "ERROR: Can't grab camera frame." << endl;
+			break;
+		}
+		/* resize to 300 x 300 */
+		cv::resize(frame, frame1, cv::Size(IMG_HEIGHT, IMG_WIDTH));
+		cv::cvtColor(frame1, rgb_mat, cv::COLOR_BGR2RGB);
+
+		if (interpreter->Invoke() != kTfLiteOk) {
+			std::cout << "Failed to invoke tflite!";
+			exit(-1);
+		}
+
+		SSDshowOutput(scores, classes, locations, &frame);
+		imshow("MoblieNetSSD-Demo", frame);
+		waitKey(1);
+	}
+	return 0;
+}