Metal每日分享，3x3矩陣卷積濾鏡效果

本案例的目的是理解如何用Metal實現3x3卷積矩陣效果濾鏡，取像素點周邊九個區域半徑點像素rgb值進行矩陣運算獲取新的rgb值;

目前有如下幾款卷積核供使用，

default：空卷積核
identity：原始卷積核
edgedetect：邊緣檢測卷積核
embossment：浮雕濾波器卷積核
embossment45：45度的浮雕濾波器卷積核
morphological：侵蝕卷積核
laplance：拉普拉斯算子，邊緣檢測算子
sharpen：銳化卷積核
sobel：邊緣提取卷積核，求梯度比較常用

效果圖

實操代碼

// 銳化卷積效果濾鏡
let filter = C7ConvolutionMatrix3x3(convolutionType: .sharpen(iterations: 2))

// 方案1:
let dest = BoxxIO.init(element: originImage, filter: filter)
ImageView.image = try? dest.output()

dest.filters.forEach {
    NSLog("%@", "\($0.parameterDescription)")
}

// 方案2:
ImageView.image = try? originImage.make(filter: filter)

// 方案3:
ImageView.image = originImage ->> filter

實現原理

過濾器

這款濾鏡採用並行計算編碼器設計.compute(kernel: "C7ConvolutionMatrix3x3")，參數因子[Float(convolutionPixel), matrix.values]

對外開放參數

convolutionPixel: 卷積像素

/// 3 x 3卷積
public struct C7ConvolutionMatrix3x3: C7FilterProtocol {

    public enum ConvolutionType {
        case `default`
        case identity
        case edgedetect
        case embossment
        case embossment45
        case morphological
        case sobel(orientation: Bool)
        case laplance(iterations: Float)
        case sharpen(iterations: Float)
        case custom(Matrix3x3)
    }

    /// Convolution pixels, default 1
    public var convolutionPixel: Int = 1
    private var matrix: Matrix3x3

    public var modifier: Modifier {
        return .compute(kernel: "C7ConvolutionMatrix3x3")
    }
    
    public var factors: [Float] {
        var array = [Float(convolutionPixel)]
        array += matrix.values
        return array
    }

    public init(matrix: Matrix3x3) {
        self.matrix = matrix
    }

    public init(convolutionType: ConvolutionType) {
        self.init(matrix: convolutionType.matrix)
    }

    public mutating func updateConvolutionType(_ convolutionType: ConvolutionType) {
        self.matrix = convolutionType.matrix
    }

    public mutating func updateMatrix3x3(_ matrix: Matrix3x3) {
        self.matrix = matrix
    }
}

extension C7ConvolutionMatrix3x3.ConvolutionType {
    var matrix: Matrix3x3 {
        switch self {
        case .identity:
            return Matrix3x3.Kernel.identity
        case .edgedetect:
            return Matrix3x3.Kernel.edgedetect
        case .embossment:
            return Matrix3x3.Kernel.embossment
        case .embossment45:
            return Matrix3x3.Kernel.embossment45
        case .morphological:
            return Matrix3x3.Kernel.morphological
        case .sobel(let orientation):
            return Matrix3x3.Kernel.sobel(orientation)
        case .laplance(let iterations):
            return Matrix3x3.Kernel.laplance(iterations)
        case .sharpen(let iterations):
            return Matrix3x3.Kernel.sharpen(iterations)
        case .custom(let matrix3x3):
            return matrix3x3
        default:
            return Matrix3x3.Kernel.`default`
        }
    }
}

着色器

取像素點周邊九個區域半徑點像素，然後歸一化處理，然後取出每個像素對應rgb，再進行卷積矩陣運算得到卷積之後的rgb值，生成新的像素顏色；

kernel void C7ConvolutionMatrix3x3(texture2d<half, access::write> outputTexture [[texture(0)]],
                                   texture2d<half, access::sample> inputTexture [[texture(1)]],
                                   constant float *pixel [[buffer(0)]],
                                   constant float *m11 [[buffer(1)]],
                                   constant float *m12 [[buffer(2)]],
                                   constant float *m13 [[buffer(3)]],
                                   constant float *m21 [[buffer(4)]],
                                   constant float *m22 [[buffer(5)]],
                                   constant float *m23 [[buffer(6)]],
                                   constant float *m31 [[buffer(7)]],
                                   constant float *m32 [[buffer(8)]],
                                   constant float *m33 [[buffer(9)]],
                                   uint2 grid [[thread_position_in_grid]]) {
    constexpr sampler quadSampler(mag_filter::linear, min_filter::linear);
    const float x = float(grid.x);
    const float y = float(grid.y);
    const float w = float(inputTexture.get_width());
    const float h = float(inputTexture.get_height());
    const float l = float(x - *pixel);
    const float r = float(x + *pixel);
    const float t = float(y - *pixel);
    const float b = float(y + *pixel);

    // Normalization
    const float2 m11Coordinate = float2(l / w, t / h);
    const float2 m12Coordinate = float2(x / w, t / h);
    const float2 m13Coordinate = float2(r / w, t / h);
    const float2 m21Coordinate = float2(l / w, y / h);
    const float2 m22Coordinate = float2(x / w, y / h);
    const float2 m23Coordinate = float2(r / w, y / h);
    const float2 m31Coordinate = float2(l / w, b / h);
    const float2 m32Coordinate = float2(x / w, b / h);
    const float2 m33Coordinate = float2(r / w, b / h);

    const half4 centerColor = inputTexture.sample(quadSampler, m22Coordinate);
    
    const half3 m11Color = inputTexture.sample(quadSampler, m11Coordinate).rgb;
    const half3 m12Color = inputTexture.sample(quadSampler, m12Coordinate).rgb;
    const half3 m13Color = inputTexture.sample(quadSampler, m13Coordinate).rgb;
    const half3 m21Color = inputTexture.sample(quadSampler, m21Coordinate).rgb;
    const half3 m22Color = centerColor.rgb;
    const half3 m23Color = inputTexture.sample(quadSampler, m23Coordinate).rgb;
    const half3 m31Color = inputTexture.sample(quadSampler, m31Coordinate).rgb;
    const half3 m32Color = inputTexture.sample(quadSampler, m32Coordinate).rgb;
    const half3 m33Color = inputTexture.sample(quadSampler, m33Coordinate).rgb;

    half3 resultColor = half3(0.0h);
    resultColor += m11Color * (*m11) + m12Color * (*m12) + m13Color * (*m13);
    resultColor += m21Color * (*m21) + m22Color * (*m22) + m23Color * (*m23);
    resultColor += m31Color * (*m31) + m32Color * (*m32) + m33Color * (*m33);

    const half4 outColor = half4(resultColor, centerColor.a);
    outputTexture.write(outColor, grid);
}

其他卷積核

extension Matrix3x3 {
    /// 常見 3x3 矩陣卷積內核，考線性代數時刻😪
    /// Common 3x3 matrix convolution kernel
    public struct Kernel { }
}

extension Matrix3x3.Kernel {
    /// 原始矩陣，空卷積核
    /// The original matrix, the empty convolution kernel
    public static let `default` = Matrix3x3(values: [
        0.0, 0.0, 0.0,
        0.0, 1.0, 0.0,
        0.0, 0.0, 0.0,
    ])

    public static let identity = Matrix3x3(values: [
        1.0, 0.0, 0.0,
        0.0, 1.0, 0.0,
        0.0, 0.0, 1.0,
    ])

    /// 邊緣檢測矩陣
    /// Edge detection matrix
    public static let edgedetect = Matrix3x3(values: [
        -1.0, -1.0, -1.0,
        -1.0,  8.0, -1.0,
        -1.0, -1.0, -1.0,
    ])

    /// 浮雕矩陣
    /// Anaglyph matrix
    public static let embossment = Matrix3x3(values: [
        -2.0, 0.0, 0.0,
         0.0, 1.0, 0.0,
         0.0, 0.0, 2.0,
    ])

    /// 45度的浮雕濾波器
    /// A 45 degree emboss filter
    public static let embossment45 = Matrix3x3(values: [
        -1.0, -1.0, 0.0,
        -1.0,  0.0, 1.0,
         0.0,  1.0, 1.0,
    ])

    /// 侵蝕矩陣
    /// Matrix erosion
    public static let morphological = Matrix3x3(values: [
        1.0, 1.0, 1.0,
        1.0, 1.0, 1.0,
        1.0, 1.0, 1.0,
    ])
    
    /// 拉普拉斯算子，邊緣檢測算子
    /// Laplace operator, edge detection operator
    public static func laplance(_ iterations: Float) -> Matrix3x3 {
        let xxx = iterations
        return Matrix3x3(values: [
             0.0, -1.0,  0.0,
            -1.0,  xxx, -1.0,
             0.0, -1.0,  0.0,
        ])
    }
    
    /// 銳化矩陣
    /// Sharpening matrix
    public static func sharpen(_ iterations: Float) -> Matrix3x3 {
        let cc = (8 * iterations + 1)
        let xx = (-iterations)
        return Matrix3x3(values: [
            xx, xx, xx,
            xx, cc, xx,
            xx, xx, xx,
        ])
    }
    
    /// Sobel矩陣圖像邊緣提取，求梯度比較常用
    /// Sobel matrix image edge extraction, gradient is more commonly used
    public static func sobel(_ orientation: Bool) -> Matrix3x3 {
        if orientation {
            return Matrix3x3(values: [
                -1.0, 0.0, 1.0,
                -2.0, 0.0, 2.0,
                -1.0, 0.0, 1.0,
            ])
        } else {
            return Matrix3x3(values: [
                -1.0, -2.0, -1.0,
                 0.0,  0.0,  0.0,
                 1.0,  2.0,  1.0,
            ])
        }
    }
    
    /// BT.601, which is the standard for SDTV.
    public static let to601 = Matrix3x3(values: [
        1.164,  1.164, 1.164,
        0.000, -0.392, 2.017,
        1.596, -0.813, 0.000,
    ])
    
    /// BT.601 full range (ref: http://www.equasys.de/colorconversion.html)
    public static let to601FullRange = Matrix3x3(values: [
        1.0,  1.000, 1.000,
        0.0, -0.343, 1.765,
        1.4, -0.711, 0.000,
    ])
    
    /// BT.709, which is the standard for HDTV.
    public static let to709 = Matrix3x3(values: [
        1.164,  1.164, 1.164,
        0.000, -0.213, 2.112,
        1.793, -0.533, 0.000,
    ])
}

Harbeth功能清單

支持ios系統和macOS系統
支持運算符函數式操作
支持多種模式數據源 UIImage, CIImage, CGImage, CMSampleBuffer, CVPixelBuffer.
支持快速設計濾鏡
支持合併多種濾鏡效果
支持輸出源的快速擴展
支持相機採集特效
支持視頻添加濾鏡特效
支持矩陣卷積
支持使用系統 MetalPerformanceShaders.
支持兼容 CoreImage.
濾鏡部分大致分爲以下幾個模塊：
- Blend：圖像融合技術
- Blur：模糊效果
- Pixel：圖像的基本像素顏色處理
- Effect：效果處理
- Lookup：查找表過濾器
- Matrix: 矩陣卷積濾波器
- Shape：圖像形狀大小相關
- Visual: 視覺動態特效
- MPS: 系統 MetalPerformanceShaders.

最後

關於3x3矩陣卷積效果濾鏡介紹與設計到此爲止吧。
慢慢再補充其他相關濾鏡，喜歡就給我點個星🌟吧。
濾鏡Demo地址，目前包含100+種濾鏡，同時也支持CoreImage混合使用。
再附上一個開發加速庫KJCategoriesDemo地址
再附上一個網絡基礎庫RxNetworksDemo地址
喜歡的老闆們可以點個星🌟，謝謝各位老闆！！！

✌️.

Metal每日分享，3x3矩陣卷積濾鏡效果

效果圖

實操代碼

實現原理

其他卷積核

Harbeth功能清單

最後

如何使用 JS 判斷用戶是否處於活躍狀態

通過HPA+CronHPA組合應對業務複雜彈性伸縮場景

Kakapos全平臺視頻導出並添加濾鏡工具

關於macOS替代品之CADisplayLink

Metal每日分享，高斯雙邊模糊濾鏡效果

Metal每日分享，自然飽和度濾鏡效果

Metal每日分享，基於色溫調整白平衡濾鏡效果

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結