stoneneedle module的安裝和使用（內核編譯）

第一次安裝內核模塊，搞了很久。有很多需要整理的，整理出這篇文章供下次使用。

剛開始安裝這個模塊的時候根據github wiki的步驟，感覺嘗試了好多次也沒有成功，主要是因爲當前系統的內核版本和編譯stoneneedle模塊的內核源碼版本不一致，所以一定要保證當前的系統內核版本和編譯stoneneedle的內核版本一致！！！要不會報錯，而且編譯不成功。

而且我嘗試過先改變系統內核版本（原先是5.2.1），通過編譯內核源碼（linux 3.13.1，3.13.0，3.16.75這幾個版本不知道爲什麼按照正常的編譯操作會報錯，所以我最後嘗試了linux 4.3.3版本終於編譯成功的！！，編譯過程見上文）。

改變系統內核爲4.3.3之後，開始之後的安裝stoneneedle操作。

git clone https://github.com/Samsung/StoneNeedle.git

然後把stoneneedle的driver/block文件下的stoneneedle.c和stoneneedle.h放到內核源碼下：

cp $sn/Stone_Needle/drivers/block/stoneneedle.c /usr/src/linux-4.3.3/drivers/block/
cp $sn/Stone_Needle/drivers/block/stoneneedle.h /usr/src/linux-4.3.3/drivers/block/

然後就是修改linux下driver/block文件夾下的幾個配置文件：

首先是Kconfig中添加：（config BLK_DEV_NVME配置下添加）：

config BLK_DEV_STONENEEDLE
        bool "STONENEEDLE module"
        depends on BLK_DEV_NVME
        ---help---
          For StoneNeedle.

然後是在Makefile文件中添加：

stoneneedle-$(CONFIG_BLK_DEV_STONENEEDLE)    := stoneneedle.o #添加整行
nvme-y		:= nvme-core.o nvme-scsi.o $(stoneneedle-y) #添加最後部分

然後在stoneneedle.c中添加下面兩行，要不會報找不到register_stoneneedle和unregister_stoneneedle函數的錯誤：

EXPORT_SYMBOL(register_stoneneedle)#加在register_stoneneedle函數外面+後面
EXPORT_SYMBOL(unregister_stoneneedle)#加在unregister_stoneneedle函數外面+後面

重點來了，4.3.3版本的linux和stoneneedle github上使用的linux版本不一樣，結果有所改變，bio結構體的參數發生了變化：

所以需要在stoneneedle.c中全文查找bio->bi_sector和bio->bi_size ，替換成bio->bi_iter.bi_sector和bio->bi_iter.bi_size。要不然編譯的時候也會報錯。報錯類似（一部分報錯）：

drivers/block/stoneneedle.c:657:17: error: ‘struct bio’ has no member named ‘bi_sector’
   int index = bio->bi_sector / bucket_size;
                  ^
 drivers/block/stoneneedle.c: In function ‘calc_write_stoneneedle’:
 drivers/block/stoneneedle.c:732:29: error: ‘struct bio’ has no member named ‘bi_size’
   unsigned int rq_bytes = bio->bi_size;

然後是修改nvme-core.c文件：（加入完整函數主要是讓大家明確stoneneedle相關函數加在nvme-core.c什麼位置）

在頭文件下面加上：
/* StoneNeedle header files, module params and global variables */
#ifdef CONFIG_BLK_DEV_STONENEEDLE
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include "stoneneedle.h"
unsigned int stoneneedle_chunk_size = 100;
EXPORT_SYMBOL_GPL(stoneneedle_chunk_size);
module_param(stoneneedle_chunk_size, int, 0644);
struct stoneneedle_ops sn_ops;
MODULE_PARM_DESC(stoneneedle_chunk_size, " 100, 1000, 10000(default 100)");
#endif /* CONFIG_BLK_DEV_STONENEEDLE */

在nvme_submit_iod函數中添加calc_stoneneedle函數：
static int nvme_submit_iod(struct nvme_queue *nvmeq, struct nvme_iod *iod,
                                                        struct nvme_ns *ns)
{
        struct request *req = iod_get_private(iod);
        struct nvme_command cmnd;
        u16 control = 0;
        u32 dsmgmt = 0;

        if (req->cmd_flags & REQ_FUA)
                control |= NVME_RW_FUA;
        if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
                control |= NVME_RW_LR;

        if (req->cmd_flags & REQ_RAHEAD)
                dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;

        memset(&cmnd, 0, sizeof(cmnd));
        cmnd.rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
        cmnd.rw.command_id = req->tag;
        cmnd.rw.nsid = cpu_to_le32(ns->ns_id);
        cmnd.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
        cmnd.rw.prp2 = cpu_to_le64(iod->first_dma);
        cmnd.rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
        cmnd.rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);

        if (ns->ms) {
                switch (ns->pi_type) {
                case NVME_NS_DPS_PI_TYPE3:
                        control |= NVME_RW_PRINFO_PRCHK_GUARD;
                        break;
                case NVME_NS_DPS_PI_TYPE1:
                case NVME_NS_DPS_PI_TYPE2:
                        control |= NVME_RW_PRINFO_PRCHK_GUARD |
                                        NVME_RW_PRINFO_PRCHK_REF;
                        cmnd.rw.reftag = cpu_to_le32(
                                        nvme_block_nr(ns, blk_rq_pos(req)));
                        break;
                }  
        if (blk_integrity_rq(req))
                        cmnd.rw.metadata =
                                cpu_to_le64(sg_dma_address(iod->meta_sg));
                else
                        control |= NVME_RW_PRINFO_PRACT;
        }

        cmnd.rw.control = cpu_to_le16(control);
        cmnd.rw.dsmgmt = cpu_to_le32(dsmgmt);
/* StoneNeedle calculation */
#ifdef CONFIG_BLK_DEV_STONENEEDLE
        if (sn_ops.calc_stoneneedle)
                sn_ops.calc_stoneneedle(ns->disk->disk_name, cmnd, req->bio);
#endif /* CONFIG_BLK_DEV_STONENEEDLE */
        __nvme_submit_cmd(nvmeq, &cmnd);

        return 0;
}


在nvme_alloc_ns函數中添加setup_stoneneedle函數：
static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
{
        struct nvme_ns *ns;
        struct gendisk *disk;
        int node = dev_to_node(dev->dev);

        ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
        if (!ns)
                return;

        ns->queue = blk_mq_init_queue(&dev->tagset);
        if (IS_ERR(ns->queue))
                goto out_free_ns;
        queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
        queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
        ns->dev = dev;
        ns->queue->queuedata = ns;

        disk = alloc_disk_node(0, node);
        if (!disk)
                goto out_free_queue;

        ns->ns_id = nsid;
        ns->disk = disk;
        ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
        list_add_tail(&ns->list, &dev->namespaces);

        blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
        if (dev->max_hw_sectors) {
                blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
                blk_queue_max_segments(ns->queue,
                        ((dev->max_hw_sectors << 9) / dev->page_size) + 1);
        }
        if (dev->stripe_size)
                blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9);
        if (dev->vwc & NVME_CTRL_VWC_PRESENT)
                blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
        blk_queue_virt_boundary(ns->queue, dev->page_size - 1);
        disk->major = nvme_major;
        disk->first_minor = 0;
        disk->fops = &nvme_fops;
        disk->private_data = ns;
        disk->queue = ns->queue;
        disk->driverfs_dev = dev->device;
        disk->flags = GENHD_FL_EXT_DEVT;
        sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid);

        /*
         * Initialize capacity to 0 until we establish the namespace format and
         * setup integrity extentions if necessary. The revalidate_disk after
         * add_disk allows the driver to register with integrity if the format
         * requires it.
         */
        set_capacity(disk, 0);
 /* Setup StoneNeedle for device */
#ifdef CONFIG_BLK_DEV_STONENEEDLE
        if (sn_ops.setup_stoneneedle) {
                if (sn_ops.setup_stoneneedle(disk, disk->disk_name) < 0)
                        pr_err("setup_stoneneedle: %s failed!!!\n",
                               disk->disk_name);
        }
#endif /* CONFIG_BLK_DEV_STONENEEDLE */
        if (nvme_revalidate_disk(ns->disk))
                goto out_free_disk;

        add_disk(ns->disk);
        if (ns->ms) {
                struct block_device *bd = bdget_disk(ns->disk, 0);
                if (!bd)
                        return;
                if (blkdev_get(bd, FMODE_READ, NULL)) {
                        bdput(bd);
                        return;
                }
                blkdev_reread_part(bd);
                blkdev_put(bd, FMODE_READ);
        }
        return; 
 out_free_disk:
        kfree(disk);
        list_del(&ns->list);
 out_free_queue:
        blk_cleanup_queue(ns->queue);
 out_free_ns:
        kfree(ns);
}


在nvme_ns_remove函數中添加release_stoneneedle函數
static void nvme_ns_remove(struct nvme_ns *ns)
{
        bool kill = nvme_io_incapable(ns->dev) && !blk_queue_dying(ns->queue);
/* Release StoneNeedle */
#ifdef CONFIG_BLK_DEV_STONENEEDLE
        if (sn_ops.release_stoneneedle)
            sn_ops.release_stoneneedle(ns->disk->disk_name);
#endif /* CONFIG_BLK_DEV_STONENEEDLE */
        if (kill)
                blk_set_queue_dying(ns->queue);
        if (ns->disk->flags & GENHD_FL_UP) {
                if (blk_get_integrity(ns->disk))
                        blk_integrity_unregister(ns->disk);
                del_gendisk(ns->disk);
        }
        if (kill || !blk_queue_dying(ns->queue)) {
                blk_mq_abort_requeue_list(ns->queue);
                blk_cleanup_queue(ns->queue);
        }
}


在nvme_init函數中添加register_stoneneedle函數
static int __init nvme_init(void)
{
        int result;

        init_waitqueue_head(&nvme_kthread_wait);

        nvme_workq = create_singlethread_workqueue("nvme");
        if (!nvme_workq)
                return -ENOMEM;

        result = register_blkdev(nvme_major, "nvme");
        if (result < 0)
                goto kill_workq;
        else if (result > 0)
                nvme_major = result;

        result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
                                                        &nvme_dev_fops);
        if (result < 0)
                goto unregister_blkdev;
        else if (result > 0)
                nvme_char_major = result;

        nvme_class = class_create(THIS_MODULE, "nvme");
        if (IS_ERR(nvme_class)) {
                result = PTR_ERR(nvme_class);
                goto unregister_chrdev;
        }
        /* StoneNeedle init */
#ifdef CONFIG_BLK_DEV_STONENEEDLE
        result = register_stoneneedle(stoneneedle_chunk_size, &sn_ops);
        if (result)
                pr_err("Register StoneNeedle failed, code: %d\n", result);
#endif  /* CONFIG_BLK_DEV_STONENEEDLE */
        result = pci_register_driver(&nvme_driver);
        if (result)
                goto destroy_class;
        return 0;
 destroy_class:
        class_destroy(nvme_class);
 unregister_chrdev:
        __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
 unregister_blkdev:
        unregister_blkdev(nvme_major, "nvme");
 kill_workq:
        destroy_workqueue(nvme_workq);
        return result;
}


在nvme_exit函數中添加unregister_stoneneedle函數：
static void __exit nvme_exit(void)
{
        pci_unregister_driver(&nvme_driver);
/* StoneNeedle exit */
#ifdef CONFIG_BLK_DEV_STONENEEDLE
        unregister_stoneneedle(&sn_ops);
#endif  /* CONFIG_BLK_DEV_STONENEEDLE */
        unregister_blkdev(nvme_major, "nvme");
        destroy_workqueue(nvme_workq);
        class_destroy(nvme_class);
        __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
        BUG_ON(nvme_thread && !IS_ERR(nvme_thread));
        _nvme_check_size();
}

然後就是編譯了,如果make install之後沒有任何warning或者是error就表示編譯成功了：

make distclean #清理環境
make menuconfig #默認設置，只是在block_driver下的NVM Express block device處修改爲<*>,在STONENEEDLE module處修改爲[*]。然後save，退出就可以啦
make -j8
make modules_install
make install

然後reboot，重啓後查看，如果下面文件夾下有stoneneedle文件夾，表示模塊安裝成功：

cd /proc

之後通過lsblk查看當前機器有的SSD，然後執行：

echo "start nvme2n1" > /proc/stoneneedle/controller  #啓動
cat /proc/stoneneedle/nvme2n1 #查看
echo "stop nvme2n1" > /proc/stoneneedle/controller  #關閉

就可以抓取相應的參數了。