NCNN

ncnn 是騰訊提供的移動端框架非常時候在手機玩

ncnn 是一個爲手機端極致優化的高性能神經網絡前向計算框架。ncnn 從設計之初深刻考慮手機端的部署和使用。無第三方依賴，跨平臺，手機端 cpu 的速度快於目前所有已知的開源框架。基於 ncnn，開發者能夠將深度學習算法輕鬆移植到手機端高效執行，開發出人工智能 APP，將 AI 帶到你的指尖。ncnn 目前已在騰訊多款應用中使用，如 QQ，Qzone，微信，天天P圖等。

功能概述

支持卷積神經網絡，支持多輸入和多分支結構，可計算部分分支
無任何第三方庫依賴，不依賴 BLAS/NNPACK 等計算框架
純 C++ 實現，跨平臺，支持 android ios 等
ARM NEON 彙編級良心優化，計算速度極快
精細的內存管理和數據結構設計，內存佔用極低
支持多核並行計算加速，ARM big.LITTLE cpu 調度優化
支持基於全新低消耗的 vulkan api GPU 加速
整體庫體積小於 700K，並可輕鬆精簡到小於 300K
可擴展的模型設計，支持 8bit 量化和半精度浮點存儲，可導入 caffe/pytorch/mxnet/onnx/darknet 模型
支持直接內存零拷貝引用加載網絡模型
可註冊自定義層實現並擴展
恩，很強就是了，不怕被塞卷 QvQ

demo功能演示

1.人臉識別

2.身份證，銀行卡識別

3.語音識別

4.車票識別

5.人體部位識別

DEMO解決了幾個問題：

1.替換了模型，檢測不出結果的問題

2.模型加載不了的問題或者奔潰

3.模型加載耗時6s。問題修復

4.識別內存溢出問題修復

demo地址：模型加載和自動識別

https://github.com/nihui/ncnn-android-mobilenetssd

3分鐘實現demo：

step1

https://github.com/Tencent/ncnn/releases

download ncnn-android-vulkan-lib.zip or build ncnn for android yourself

下載庫ncnn-android-vulkan-lib：

step2

extract ncnn-android-vulkan-lib.zip into app/src/main/jni or change the ncnn path to yours in app/src/main/jni/CMakeLists.txt

配置cmake

android ios 預編譯庫 20200616 622879a

編譯版本，默認配置，android-ndk-r21d，cctools-port 895 + ld64-274.2 + ios 10.2 sdk libc++
ncnn-android-lib 是 android 的靜態庫(armeabi-v7a + arm64-v8a + x86 + x86_64)
ncnn-android-vulkan-lib 是 android 的靜態庫(armeabi-v7a + arm64-v8a + x86 + x86_64，包含vulkan支持)
ncnn.framework.zip 是 ios 的靜態庫(armv7 + arm64 + i386 + x86_64，bitcode)
ncnn-vulkan.framework.zip 是 ios 的靜態庫(arm64 + x86_64，bitcode，包含vulkan支持，MoltenVK-1.1.82.0)
openmp.framework.zip 是 ios ncnn openmp 運行時靜態庫(armv7 + arm64 + i386 + x86_64，bitcode)

adreno gpu image存儲+fp16p/fp16s/fp16pa/fp16sa優化，在qcom855之前的高通芯片上默認啓用，包括全部gpu shader
新增darknet轉換器，支持yolov4和efficientnetb0-yolov3(by zhiliu6)
新增simplestl，可替代std::string/std::vector，默認不啓用(by scarsty)
新增NCNN_LOGE宏，android自動在adb logcat輸出信息(by maxint)
運行時生成spirv，大幅減小gpu庫體積
新增python綁定鏈接
新增查詢當前可用gpu顯存接口
gpu fp16/fp32轉換，buffer/image

源碼分析：

第一步：加載模型。量化模型

// init param
{
    int ret = mobilenetssd.load_param(mgr, "mobilenet_ssd_voc_ncnn.param");
    if (ret != 0)
    {
        __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_param failed");
        return JNI_FALSE;
    }
    __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_param success");
}

// init bin
{
    int ret = mobilenetssd.load_model(mgr, "mobilenet_ssd_voc_ncnn.bin");
    if (ret != 0)
    {
        __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_model failed");
        return JNI_FALSE;
    }

    __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_model success");
}

第二步：初始化java需要的一些參數

// init jni glue
jclass localObjCls = env->FindClass("com/tencent/mobilenetssdncnn/MobilenetSSDNcnn$Obj");
objCls = reinterpret_cast<jclass>(env->NewGlobalRef(localObjCls));

__android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "start");

constructortorId = env->GetMethodID(objCls, "<init>", "(Lcom/tencent/mobilenetssdncnn/MobilenetSSDNcnn;)V");

xId = env->GetFieldID(objCls, "x", "F");
yId = env->GetFieldID(objCls, "y", "F");
wId = env->GetFieldID(objCls, "w", "F");
hId = env->GetFieldID(objCls, "h", "F");
labelId = env->GetFieldID(objCls, "label", "Ljava/lang/String;");
probId = env->GetFieldID(objCls, "prob", "F");

__android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "end");

第三步：識別模型

}

double start_time = ncnn::get_current_time();

AndroidBitmapInfo info;
AndroidBitmap_getInfo(env, bitmap, &info);
int width = info.width;
int height = info.height;
if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888)
    return NULL;

// ncnn from bitmap
ncnn::Mat in = ncnn::Mat::from_android_bitmap_resize(env, bitmap, ncnn::Mat::PIXEL_BGR, 300, 300);

// mobilenetssd
std::vector<Object> objects;
{
    const float mean_vals[3] = { 98.1f, 106.0f, 133.1f };
    const float norm_vals[3] = { 0.2207f, 0.2281f, 0.2640f };
    in.substract_mean_normalize(mean_vals, norm_vals);

    ncnn::Extractor ex = mobilenetssd.create_extractor();

    ex.set_vulkan_compute(use_gpu);

    ex.input("data", in);

    ncnn::Mat out;
    ex.extract("detection_out", out);

    for (int i=0; i<out.h; i++)
    {
        const float* values = out.row(i);
        if(values[0]==1&&values[1]>0.7){
            Object object;
            object.label = values[0];
            object.prob = values[1];
            object.x = values[2] * width;
            object.y = values[3] * height;
            object.w = values[4] * width - object.x;
            object.h = values[5] * height - object.y;

            objects.push_back(object);
        } else{
            continue;
        }

    }
}

第4步：根據座標進行車子的繪製

    for (int i = 0; i < objects.length; i++)

    {

        Log.d("peng","objects"+objects[i].toString());
        canvas.drawRect(objects[i].x, objects[i].y, objects[i].x + objects[i].w, objects[i].y + objects[i].h, paint);

        // draw filled text inside image
        {
            String text = objects[i].label + " = " + String.format("%.1f", objects[i].prob * 100) + "%";

            float text_width = textpaint.measureText(text);
            float text_height = - textpaint.ascent() + textpaint.descent();

            float x = objects[i].x;
            float y = objects[i].y - text_height;
            if (y < 0)
                y = 0;
            if (x + text_width > rgba.getWidth())
                x = rgba.getWidth() - text_width;

            canvas.drawRect(x, y, x + text_width, y + text_height, textbgpaint);

            canvas.drawText(text, x, y - textpaint.ascent(), textpaint);
        }
    }

    imageView.setImageBitmap(rgba);
}

#include <android/asset_manager_jni.h>
#include <android/bitmap.h>
#include <android/log.h>

#include <jni.h>

#include <string>
#include <vector>
#include <gpu.h>

// ncnn
#include "net.h"
#include "benchmark.h"

static ncnn::UnlockedPoolAllocator g_blob_pool_allocator;
static ncnn::PoolAllocator g_workspace_pool_allocator;

static ncnn::Net mobilenetssd;

struct Object
{
    float x;
    float y;
    float w;
    float h;
    int label;
    float prob;
};

extern "C" {

// FIXME DeleteGlobalRef is missing for objCls
static jclass objCls = NULL;
static jmethodID constructortorId;
static jfieldID xId;
static jfieldID yId;
static jfieldID wId;
static jfieldID hId;
static jfieldID labelId;
static jfieldID probId;

JNIEXPORT jint JNI_OnLoad(JavaVM* vm, void* reserved)
{
    __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "JNI_OnLoad");

    ncnn::create_gpu_instance();

    return JNI_VERSION_1_4;
}

JNIEXPORT void JNI_OnUnload(JavaVM* vm, void* reserved)
{
    __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "JNI_OnUnload");

    ncnn::destroy_gpu_instance();
}

// public native boolean Init(AssetManager mgr);
JNIEXPORT jboolean JNICALL Java_com_tencent_mobilenetssdncnn_MobilenetSSDNcnn_Init(JNIEnv* env, jobject thiz, jobject assetManager)
{
    ncnn::Option opt;
    opt.lightmode = true;
    opt.num_threads = 4;
    opt.blob_allocator = &g_blob_pool_allocator;
    opt.workspace_allocator = &g_workspace_pool_allocator;
    opt.use_packing_layout = true;

    // use vulkan compute
    if (ncnn::get_gpu_count() != 0)
        opt.use_vulkan_compute = true;

    AAssetManager* mgr = AAssetManager_fromJava(env, assetManager);

    mobilenetssd.opt = opt;

    // init param
    {
        int ret = mobilenetssd.load_param(mgr, "mobilenet_ssd_voc_ncnn.param");
        if (ret != 0)
        {
            __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_param failed");
            return JNI_FALSE;
        }
        __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_param success");
    }

    // init bin
    {
        int ret = mobilenetssd.load_model(mgr, "mobilenet_ssd_voc_ncnn.bin");
        if (ret != 0)
        {
            __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_model failed");
            return JNI_FALSE;
        }

        __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_model success");
    }

    // init jni glue
    jclass localObjCls = env->FindClass("com/tencent/mobilenetssdncnn/MobilenetSSDNcnn$Obj");
    objCls = reinterpret_cast<jclass>(env->NewGlobalRef(localObjCls));

    __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "start");

    constructortorId = env->GetMethodID(objCls, "<init>", "(Lcom/tencent/mobilenetssdncnn/MobilenetSSDNcnn;)V");

    xId = env->GetFieldID(objCls, "x", "F");
    yId = env->GetFieldID(objCls, "y", "F");
    wId = env->GetFieldID(objCls, "w", "F");
    hId = env->GetFieldID(objCls, "h", "F");
    labelId = env->GetFieldID(objCls, "label", "Ljava/lang/String;");
    probId = env->GetFieldID(objCls, "prob", "F");

    __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "end");

    return JNI_TRUE;
}

// public native Obj[] Detect(Bitmap bitmap, boolean use_gpu);
JNIEXPORT jobjectArray JNICALL Java_com_tencent_mobilenetssdncnn_MobilenetSSDNcnn_Detect(JNIEnv* env, jobject thiz, jobject bitmap, jboolean use_gpu)
{
    if (use_gpu == JNI_TRUE && ncnn::get_gpu_count() == 0)
    {
        return NULL;
        //return env->NewStringUTF("no vulkan capable gpu");
    }

    double start_time = ncnn::get_current_time();

    AndroidBitmapInfo info;
    AndroidBitmap_getInfo(env, bitmap, &info);
    int width = info.width;
    int height = info.height;
    if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888)
        return NULL;

    // ncnn from bitmap
    ncnn::Mat in = ncnn::Mat::from_android_bitmap_resize(env, bitmap, ncnn::Mat::PIXEL_BGR, 300, 300);

    // mobilenetssd
    std::vector<Object> objects;
    {
        const float mean_vals[3] = { 98.1f, 106.0f, 133.1f };
        const float norm_vals[3] = { 0.2207f, 0.2281f, 0.2640f };
        in.substract_mean_normalize(mean_vals, norm_vals);

        ncnn::Extractor ex = mobilenetssd.create_extractor();

        ex.set_vulkan_compute(use_gpu);

        ex.input("data", in);

        ncnn::Mat out;
        ex.extract("detection_out", out);

        for (int i=0; i<out.h; i++)
        {
            const float* values = out.row(i);
            if(values[0]==1&&values[1]>0.7){
                Object object;
                object.label = values[0];
                object.prob = values[1];
                object.x = values[2] * width;
                object.y = values[3] * height;
                object.w = values[4] * width - object.x;
                object.h = values[5] * height - object.y;

                objects.push_back(object);
            } else{
                continue;
            }

        }
    }

    // objects to Obj[]
    static const char* class_names[] = {"background",
        "tongue", "furred-tongue", "tooth-mark", "crack"};

    jobjectArray jObjArray = env->NewObjectArray(objects.size(), objCls, NULL);

    for (size_t i=0; i<objects.size(); i++)
    {
        jobject jObj = env->NewObject(objCls, constructortorId, thiz);

        env->SetFloatField(jObj, xId, objects[i].x);
        env->SetFloatField(jObj, yId, objects[i].y);
        env->SetFloatField(jObj, wId, objects[i].w);
        env->SetFloatField(jObj, hId, objects[i].h);
        env->SetObjectField(jObj, labelId, env->NewStringUTF(class_names[objects[i].label]));
        env->SetFloatField(jObj, probId, objects[i].prob);

        env->SetObjectArrayElement(jObjArray, i, jObj);
    }

    double elasped = ncnn::get_current_time() - start_time;
    __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "%.2fms   detect", elasped);

    return jObjArray;
}

}

1.Android 騰訊NCNN3分鐘實現攝像頭檢測模型加載人體部位識別 (人工智能)

NCNN

功能概述

step1

step2

2.Android OpenCV 3分種實現身份證識別臉識別圖片分類圖片特效(人工智能)

4.NDK Android jni開發 C++ 基本語法 (相機圖片美化)

5.NDK Android jni開發異常處理 native奔潰解決(相機圖片美化)

6.NDK Android jni開發 so庫奔潰解決辦法 (相機圖片美化)

1.Android 騰訊NCNN3分鐘實現攝像頭檢測模型加載人體部位識別 (人工智能)

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結

1.Android 騰訊NCNN3分鐘實現 攝像頭檢測 模型加載 人體部位識別 (人工智能)

NCNN

功能概述

step1

step2

1.Android 騰訊NCNN3分鐘實現攝像頭檢測模型加載人體部位識別 (人工智能)