文件系統核心存儲引擎_實現(四)

索引文件核心頭文件定義 - Blocklnfo

#ifndef _COMMON_H_INCLUDED_
#define _COMMON_H_INCLUDED_

#include <iostream>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <string>
#include <string.h>
#include <stdint.h>
#include <errno.h>
#include <stdio.h>
#include <sys/mman.h>
#include <unistd.h>
#include <stdlib.h>
#include <inttypes.h>

namespace qiniu
{
    namespace largefile
    {
        const int32_t TFS_SUCCESS = 0;
        const int32_t TFS_ERROR = -1;
        const int32_t EXIT_DISK_OPER_INCOMPLETE = -8012; /* read or write length is less than required */

        struct Blockinfo
        {
            uint32_t blosk_id_;
            int32_t version_;
            int32_t file_count_;
            int32_t size_t;
            int32_t del_file_count_;
            int32_t del_size_;
            uint32_t seq_no_;

            Blockinfo()
            {
                memset(this, 0, sizeof(Blockinfo));
            }

            inline bool operator==(const Blockinfo &rhs) const
            {
                return blosk_id_ == rhs.blosk_id_ && version_ == rhs.version_ && file_count_ == rhs.file_count_
                && size_t == rhs.size_t && del_file_count_ == rhs.del_file_count_ && del_size_ == rhs.del_size_
                && seq_no_ == rhs.seq_no_;
            }
        };
    }
}

#endif /* _COMMON_H_INCLUDED_ */

索引文件核心頭文件定義 - Metalnfo

#ifndef _COMMON_H_INCLUDED_
#define _COMMON_H_INCLUDED_

#include <iostream>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <string>
#include <string.h>
#include <stdint.h>
#include <errno.h>
#include <stdio.h>
#include <sys/mman.h>
#include <unistd.h>
#include <stdlib.h>
#include <inttypes.h>

namespace qiniu
{
    namespace largefile
    {
        const int32_t TFS_SUCCESS = 0;
        const int32_t TFS_ERROR = -1;
        const int32_t EXIT_DISK_OPER_INCOMPLETE = -8012; /* read or write length is less than required */

        struct MMapOption
        {
            int32_t max_mmap_size_;
            int32_t first_mmap_size_;
            int32_t per_mmap_size_;
        };

        struct Blockinfo
        {
            uint32_t blosk_id_;
            int32_t version_;
            int32_t file_count_;
            int32_t size_t;
            int32_t del_file_count_;
            int32_t del_size_;
            uint32_t seq_no_;

            Blockinfo()
            {
                memset(this, 0, sizeof(Blockinfo));
            }

            inline bool operator==(const Blockinfo &rhs) const
            {
                return blosk_id_ == rhs.blosk_id_ && version_ == rhs.version_ && file_count_ == rhs.file_count_
                && size_t == rhs.size_t && del_file_count_ == rhs.del_file_count_ && del_size_ == rhs.del_size_
                && seq_no_ == rhs.seq_no_;
            }
        };


        struct Metalnfo
        {
            public:
            Metalnfo()
            {
                init();
            }

            Metalnfo(const uint64_t file_id, const int32_t in_offset, const int32_t file_size, const int32_t next_meta_offset)
            {
                fileid_ = file_id;
                location_.inner_offset_ = in_offset;
                location_.size_ = file_size;
                next_meta_offset_ = next_meta_offset;
            }

            Metalnfo(const Metalnfo &meta_info)
            {
                memcpy(this, &meta_info, sizeof(Metalnfo));
            }

            Metalnfo &operator=(const Metalnfo &meta_info)
            {
                if (this == &meta_info)
                {
                    return *this;
                }

                fileid_ = meta_info.fileid_;
                location_.inner_offset_ = meta_info.location_.inner_offset_;
                location_.size_ = meta_info.location_.size_;
                next_meta_offset_ = meta_info.next_meta_offset_;
            }

            Metalnfo &clone(const Metalnfo &meta_info)
            {
                assert(this != &meta_info);

                fileid_ = meta_info.fileid_;
                location_.inner_offset_ = meta_info.location_.inner_offset_;
                location_.size_ = meta_info.location_.size_;
                next_meta_offset_ = meta_info.next_meta_offset_;

                return *this;
            }

            bool operator == (const Metalnfo &rhs) const
            {
                return fileid_ == rhs.fileid_ && 
                location_.inner_offset_ == rhs.location_.inner_offset_ &&
                location_.size_ == rhs.location_.size_ &&
                next_meta_offset_ == rhs.next_meta_offset_;
            }

            uint64_t get_key() const
            {
                return fileid_;
            }

            void set_key(cosnt uint64_t key)
            {
                fileid_ = key;
            }

            uint64_t get_file_id() const
            {
                return fileid_;
            }

            void set_file_id(const uint64_t file_id)
            {
                fileid_ = file_id;
            }

            int32_t get_offset() const
            {
                return location_.inner_offset_;
            }

            void set_offset(const int32_t offset)
            {
                location_.inner_offset_ = offset;
            }

            int32_t get_size() const
            {
                return location_.size_;
            }

            void set_size(const int32_t file_size)
            {
                location_.size_ = file_size;
            }

            int32_t get_next_meta_offset() const
            {
                return next_meta_offset_;
            }


            private:
            uint64_t fileid_;

            struct{
                int32_t inner_offset_;
                int32_t size_;
            }location_;

            int32_t next_meta_offset_;


            private:
            void init()
            {
                fileid_ = 0;
                location_.inner_offset_ = 0;
                location_.size_ = 0;
            }
        };
    }
}
#endif /* _COMMON_H_INCLUDED_ */

面向對象分析概述 - 設計類圖

設計類圖

索引處理類頭文件定義 _IndexHeader

#ifndef QINIU_LARGEFILE_INDEX_HANDLE_H_
#define QINIU_LARGEFILE_INDEX_HANDLE_H_

#include "common.h"
#include "mmap_file_op.h"

namespace qiniu
{
    namespace largefile
    {
        struct IndexHeader
        {
        	public:
            IndexHeader()
            {
                memset(this, 0, sizeof(IndexHeader));
            }

            Blockinfo block_info_; /* meta block info */
            int32_t bucket_size; /* hash bucket size */
            int32_t data_file_offset_; /* offset to write next data in block */
            int32_t index_file_size_; /* offset after index_header + all bucketa */
            int32_t free_head_offset_; /* free meta node list, for reuse */
        };
    }
}
#endif /* QINIU_LARGEFILE_INDEX_HANDLE_H_ */

索引處理類 - 構造與析構

#include "common.h"
#include "index_handle.h"

namespace qiniu
{
    namespace largfile
    {
        /* IndexHandle(const std::string &base_path, const uint32_t main_block_id); */
        IndexHandle::IndexHandle(const std::string &base_path, const uint32_t main_block_id)
        {
            /* create file_op_ handle object */
            std::stringstream tmp_stream;
            tmp_stream << base_path << INDEX_DIR_PREFIX << main_block_id; /* /root/weifc/index/ */

            std::string index_path;
            tmp_stream >> index_path;

            file_op_ = new MMapFileOperation(index_path, O_CREAT | O_RDWR | O_LARGEFILE);
            is_load_ = false;
        }


        IndexHandle::~IndexHandle()
        {
            if (file_op_)
            {
                delete file_op_;
                file_op_ = NULL;
            }
        }

    }
}

#ifndef QINIU_LARGEFILE_INDEX_HANDLE_H_
#define QINIU_LARGEFILE_INDEX_HANDLE_H_

#include "common.h"
#include "mmap_file_op.h"

namespace qiniu
{
    namespace largefile
    {
        struct IndexHeader
        {
            public:
            IndexHeader()
            {
                memset(this, 0, sizeof(IndexHeader));
            }

            Blockinfo block_info_; /* meta block info */
            int32_t bucket_size; /* hash bucket size */
            int32_t data_file_offset_; /* offset to write next data in block */
            int32_t index_file_size_; /* offset after index_header + all bucketa */
            int32_t free_head_offset_; /* free meta node list, for reuse */
        };


        class IndexHandle
        {
            public:
            IndexHandle(const std::string &base_path, const uint32_t main_block_id);
            ~IndexHandle();
            
            private:
            MMapFileOperation *file_op;
            bool is_load_;
        }
    }
}
#endif /* QINIU_LARGEFILE_INDEX_HANDLE_H_ */

#ifndef _COMMON_H_INCLUDED_
#define _COMMON_H_INCLUDED_

#include <iostream>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <string>
#include <string.h>
#include <stdint.h>
#include <errno.h>
#include <stdio.h>
#include <sys/mman.h>
#include <unistd.h>
#include <stdlib.h>
#include <inttypes.h>

namespace qiniu
{
    namespace largefile
    {
        const int32_t TFS_SUCCESS = 0;
        const int32_t TFS_ERROR = -1;
        const int32_t EXIT_DISK_OPER_INCOMPLETE = -8012; /* read or write length is less than required */

        static const std::string MAINBLOCK_DIR_PREFIX = "/mainblock/";
        static const std::string INDEX_DIR_PREFIX = "/index/";
        static const mode_t DIR_MODE = 0755;

        struct MMapOption
        {
            int32_t max_mmap_size_;
            int32_t first_mmap_size_;
            int32_t per_mmap_size_;
        };

        struct Blockinfo
        {
            uint32_t blosk_id_;
            int32_t version_;
            int32_t file_count_;
            int32_t size_t;
            int32_t del_file_count_;
            int32_t del_size_;
            uint32_t seq_no_;

            Blockinfo()
            {
                memset(this, 0, sizeof(Blockinfo));
            }

            inline bool operator==(const Blockinfo &rhs) const
            {
                return blosk_id_ == rhs.blosk_id_ && version_ == rhs.version_ && file_count_ == rhs.file_count_
                && size_t == rhs.size_t && del_file_count_ == rhs.del_file_count_ && del_size_ == rhs.del_size_
                && seq_no_ == rhs.seq_no_;
            }
        };


        struct Metalnfo
        {
            public:
            Metalnfo()
            {
                init();
            }

            Metalnfo(const uint64_t file_id, const int32_t in_offset, const int32_t file_size, const int32_t next_meta_offset)
            {
                fileid_ = file_id;
                location_.inner_offset_ = in_offset;
                location_.size_ = file_size;
                next_meta_offset_ = next_meta_offset;
            }

            Metalnfo(const Metalnfo &meta_info)
            {
                memcpy(this, &meta_info, sizeof(Metalnfo));
            }

            Metalnfo &operator=(const Metalnfo &meta_info)
            {
                if (this == &meta_info)
                {
                    return *this;
                }

                fileid_ = meta_info.fileid_;
                location_.inner_offset_ = meta_info.location_.inner_offset_;
                location_.size_ = meta_info.location_.size_;
                next_meta_offset_ = meta_info.next_meta_offset_;
            }

            Metalnfo &clone(const Metalnfo &meta_info)
            {
                assert(this != &meta_info);

                fileid_ = meta_info.fileid_;
                location_.inner_offset_ = meta_info.location_.inner_offset_;
                location_.size_ = meta_info.location_.size_;
                next_meta_offset_ = meta_info.next_meta_offset_;

                return *this;
            }

            bool operator == (const Metalnfo &rhs) const
            {
                return fileid_ == rhs.fileid_ && 
                location_.inner_offset_ == rhs.location_.inner_offset_ &&
                location_.size_ == rhs.location_.size_ &&
                next_meta_offset_ == rhs.next_meta_offset_;
            }

            uint64_t get_key() const
            {
                return fileid_;
            }

            void set_key(cosnt uint64_t key)
            {
                fileid_ = key;
            }

            uint64_t get_file_id() const
            {
                return fileid_;
            }

            void set_file_id(const uint64_t file_id)
            {
                fileid_ = file_id;
            }

            int32_t get_offset() const
            {
                return location_.inner_offset_;
            }

            void set_offset(const int32_t offset)
            {
                location_.inner_offset_ = offset;
            }

            int32_t get_size() const
            {
                return location_.size_;
            }

            void set_size(const int32_t file_size)
            {
                location_.size_ = file_size;
            }

            int32_t get_next_meta_offset() const
            {
                return next_meta_offset_;
            }


            private:
            uint64_t fileid_;

            struct{
                int32_t inner_offset_;
                int32_t size_;
            }location_;

            int32_t next_meta_offset_;


            private:
            void init()
            {
                fileid_ = 0;
                location_.inner_offset_ = 0;
                location_.size_ = 0;
            }
        };
    }
}

#endif /* _COMMON_H_INCLUDED_ */

索引處理類 - create 方法實現

#ifndef QINIU_LARGEFILE_INDEX_HANDLE_H_
#define QINIU_LARGEFILE_INDEX_HANDLE_H_

#include "common.h"
#include "mmap_file_op.h"

namespace qiniu
{
    namespace largefile
    {
        struct IndexHeader
        {
            public:
            IndexHeader()
            {
                memset(this, 0, sizeof(IndexHeader));
            }

            Blockinfo block_info_; /* meta block info */
            int32_t bucket_size_; /* hash bucket size */
            int32_t data_file_offset_; /* offset to write next data in block */
            int32_t index_file_size_; /* offset after index_header + all bucketa */
            int32_t free_head_offset_; /* free meta node list, for reuse */
        };


        class IndexHandle
        {
            public:
            IndexHandle(const std::string &base_path, const uint32_t main_block_id);
            ~IndexHandle();

            int create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont);

            IndexHandle *index_header()
            {
                return reinterpret_cast<Indexheader*>(file_op_->get_map_data());
            }
            
            private:
            MMapFileOperation *file_op_;
            bool is_load_;
        }
    }
}
#endif /* QINIU_LARGEFILE_INDEX_HANDLE_H_ */

#include "common.h"
#include "index_handle.h"

static int debug = 1;

namespace qiniu
{
    namespace largfile
    {
        /* IndexHandle(const std::string &base_path, const uint32_t main_block_id); */
        IndexHandle::IndexHandle(const std::string &base_path, const uint32_t main_block_id)
        {
            /* create file_op_ handle object */
            std::stringstream tmp_stream;
            tmp_stream << base_path << INDEX_DIR_PREFIX << main_block_id; /* /root/weifc/index/ */

            std::string index_path;
            tmp_stream >> index_path;

            file_op_ = new MMapFileOperation(index_path, O_CREAT | O_RDWR | O_LARGEFILE);
            is_load_ = false;
        }


        IndexHandle::~IndexHandle()
        {
            if (file_op_)
            {
                delete file_op_;
                file_op_ = NULL;
            }
        }


        int IndexHandle::create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont)
        {
            int ret = TFS_SUCCESS;

            if (debug)
            {
                printf("create index, block id: %u, bucket size: %d, max_mmap_size: %d, first mmap size: %d, per mmap size: %d\n",
                        logic_block_id, bucket_size, map_optiont.max_mmap_size_, map_optiont.first_mmap_size_, map_optiont.per_mmap_size_);
            }

            if (is_load_)
            {
                return EXIT_INDEX_ALREADY_LOADED_ERROR;
            }

            int64_t file_size = file_op_->get_file_size();

            if (file_size < 0)
            {
                return TFS_ERROR;
            }else (file_size == 0) /* empty file */
            {
                IndexHandle i_header;
                i_header.block_info_.blosk_id_ = logic_block_id;
                i_header.block_info_.seq_no_ = 1;
                i_header.bucket_size_ = bucket_size;

                i_header.index_file_size_ = sizeof(IndexHandle) + bucket_size * sizeof(int32_t);

                /* index header + total buckets */
                char *init_data = new char[i_header.index_file_size_];
                memcpy(init_data, &i_header, sizeof(IndexHandle));
                memset(init_data + sizeof(IndexHandle), 0, i_header.index_file_size_ - sizeof(IndexHandle));

                /* write index header and buckets into file */
                ret = file_op_->pwrite_file(init_data, i_header.index_file_size_, 0);

                delete []init_data;
                init_data = NULL;

                if (ret != TFS_ERROR)
                {
                    return ret;
                }

                ret = file_op_->flush_file();
                if (ret != TFS_SUCCESS)
                {
                    return ret;
                }
            }else /* file size > 0, index already exist */
            {
                return EXIT_META_UNEXPECT_FOUND_ERROR;
            }
            
            ret = file_op_->mmap_file(map_optiont);
            if (ret != TFS_SUCCESS)
            {
                return ret;
            }
            
            is_load_ = true;

            if (debug)
            {
                printf("init blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, filecount: %d, det_size: %d, del_file_count: %d, version: %d\n", 
                logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
                index_header()->bucket_size_, index_header()->free_head_offset_(), block_info()->seq_no_, block_info()->size_,
                block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
            }

            return TFS_SUCCESS;
        }

    }
}

索引處理類 - load 方法實現

#ifndef _COMMON_H_INCLUDED_
#define _COMMON_H_INCLUDED_

#include <iostream>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <string>
#include <string.h>
#include <stdint.h>
#include <errno.h>
#include <stdio.h>
#include <sys/mman.h>
#include <unistd.h>
#include <stdlib.h>
#include <inttypes.h>

namespace qiniu
{
    namespace largefile
    {
        const int32_t TFS_SUCCESS = 0;
        const int32_t TFS_ERROR = -1;
        const int32_t EXIT_DISK_OPER_INCOMPLETE = -8012; /* read or write length is less than required */
        const int32_t EXIT_INDEX_ALREADY_LOADED_ERROR = -8013; /* index is loaded when create or load */
        const int32_t EXIT_META_UNEXPECT_FOUND_ERROR = -8014; /* meta fount in index when inset */
        const int32_t EXIT_INDEX_CORRUPT_ERROR = -8015; /* index is corrupt. */
        const int32_t EXIT_BLOCKID_CONFLICT_ERROR = -8016;
        const int32_t EXIT_BUCKET_CONFIGURE_ERROR = -8017;

        static const std::string MAINBLOCK_DIR_PREFIX = "/mainblock/";
        static const std::string INDEX_DIR_PREFIX = "/index/";
        static const mode_t DIR_MODE = 0755;

        struct MMapOption
        {
            int32_t max_mmap_size_;
            int32_t first_mmap_size_;
            int32_t per_mmap_size_;
        };

        struct Blockinfo
        {
            uint32_t block_id_;
            int32_t version_;
            int32_t file_count_;
            int32_t size_;
            int32_t del_file_count_;
            int32_t del_size_;
            uint32_t seq_no_;

            Blockinfo()
            {
                memset(this, 0, sizeof(Blockinfo));
            }

            inline bool operator==(const Blockinfo &rhs) const
            {
                return blosk_id_ == rhs.blosk_id_ && version_ == rhs.version_ && file_count_ == rhs.file_count_
                && size_t == rhs.size_t && del_file_count_ == rhs.del_file_count_ && del_size_ == rhs.del_size_
                && seq_no_ == rhs.seq_no_;
            }
        };


        struct Metalnfo
        {
            public:
            Metalnfo()
            {
                init();
            }

            Metalnfo(const uint64_t file_id, const int32_t in_offset, const int32_t file_size, const int32_t next_meta_offset)
            {
                fileid_ = file_id;
                location_.inner_offset_ = in_offset;
                location_.size_ = file_size;
                next_meta_offset_ = next_meta_offset;
            }

            Metalnfo(const Metalnfo &meta_info)
            {
                memcpy(this, &meta_info, sizeof(Metalnfo));
            }

            Metalnfo &operator=(const Metalnfo &meta_info)
            {
                if (this == &meta_info)
                {
                    return *this;
                }

                fileid_ = meta_info.fileid_;
                location_.inner_offset_ = meta_info.location_.inner_offset_;
                location_.size_ = meta_info.location_.size_;
                next_meta_offset_ = meta_info.next_meta_offset_;
            }

            Metalnfo &clone(const Metalnfo &meta_info)
            {
                assert(this != &meta_info);

                fileid_ = meta_info.fileid_;
                location_.inner_offset_ = meta_info.location_.inner_offset_;
                location_.size_ = meta_info.location_.size_;
                next_meta_offset_ = meta_info.next_meta_offset_;

                return *this;
            }

            bool operator == (const Metalnfo &rhs) const
            {
                return fileid_ == rhs.fileid_ && 
                location_.inner_offset_ == rhs.location_.inner_offset_ &&
                location_.size_ == rhs.location_.size_ &&
                next_meta_offset_ == rhs.next_meta_offset_;
            }

            uint64_t get_key() const
            {
                return fileid_;
            }

            void set_key(cosnt uint64_t key)
            {
                fileid_ = key;
            }

            uint64_t get_file_id() const
            {
                return fileid_;
            }

            void set_file_id(const uint64_t file_id)
            {
                fileid_ = file_id;
            }

            int32_t get_offset() const
            {
                return location_.inner_offset_;
            }

            void set_offset(const int32_t offset)
            {
                location_.inner_offset_ = offset;
            }

            int32_t get_size() const
            {
                return location_.size_;
            }

            void set_size(const int32_t file_size)
            {
                location_.size_ = file_size;
            }

            int32_t get_next_meta_offset() const
            {
                return next_meta_offset_;
            }


            private:
            uint64_t fileid_;

            struct{
                int32_t inner_offset_;
                int32_t size_;
            }location_;

            int32_t next_meta_offset_;

            private:
            void init()
            {
                fileid_ = 0;
                location_.inner_offset_ = 0;
                location_.size_ = 0;
            }
        };
    }
}

#endif /* _COMMON_H_INCLUDED_ */

#include "common.h"
#include "index_handle.h"

static int debug = 1;

namespace qiniu
{
    namespace largfile
    {
        /* IndexHandle(const std::string &base_path, const uint32_t main_block_id); */
        IndexHandle::IndexHandle(const std::string &base_path, const uint32_t main_block_id)
        {
            /* create file_op_ handle object */
            std::stringstream tmp_stream;
            tmp_stream << base_path << INDEX_DIR_PREFIX << main_block_id; /* /root/weifc/index/ */

            std::string index_path;
            tmp_stream >> index_path;

            file_op_ = new MMapFileOperation(index_path, O_CREAT | O_RDWR | O_LARGEFILE);
            is_load_ = false;
        }


        IndexHandle::~IndexHandle()
        {
            if (file_op_)
            {
                delete file_op_;
                file_op_ = NULL;
            }
        }


        int IndexHandle::create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont)
        {
            int ret = TFS_SUCCESS;

            if (debug)
            {
                printf("create index, block id: %u, bucket size: %d, max_mmap_size: %d, first mmap size: %d, per mmap size: %d\n",
                        logic_block_id, bucket_size, map_optiont.max_mmap_size_, map_optiont.first_mmap_size_, map_optiont.per_mmap_size_);
            }

            if (is_load_)
            {
                return EXIT_INDEX_ALREADY_LOADED_ERROR;
            }

            int64_t file_size = file_op_->get_file_size();

            if (file_size < 0)
            {
                return TFS_ERROR;
            }else (file_size == 0) /* empty file */
            {
                IndexHandle i_header;
                i_header.block_info_.blosk_id_ = logic_block_id;
                i_header.block_info_.seq_no_ = 1;
                i_header.bucket_size_ = bucket_size;

                i_header.index_file_size_ = sizeof(IndexHandle) + bucket_size * sizeof(int32_t);

                /* index header + total buckets */
                char *init_data = new char[i_header.index_file_size_];
                memcpy(init_data, &i_header, sizeof(IndexHandle));
                memset(init_data + sizeof(IndexHandle), 0, i_header.index_file_size_ - sizeof(IndexHandle));

                /* write index header and buckets into file */
                ret = file_op_->pwrite_file(init_data, i_header.index_file_size_, 0);

                delete []init_data;
                init_data = NULL;

                if (ret != TFS_ERROR)
                {
                    return ret;
                }

                ret = file_op_->flush_file();
                if (ret != TFS_SUCCESS)
                {
                    return ret;
                }
            }else /* file size > 0, index already exist */
            {
                return EXIT_META_UNEXPECT_FOUND_ERROR;
            }
            
            ret = file_op_->mmap_file(map_optiont);
            if (ret != TFS_SUCCESS)
            {
                return ret;
            }
            
            is_load_ = true;

            if (debug)
            {
                printf("init blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, filecount: %d, det_size: %d, del_file_count: %d, version: %d\n", 
                logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
                index_header()->bucket_size_, index_header()->free_head_offset_(), block_info()->seq_no_, block_info()->size_,
                block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
            }

            return TFS_SUCCESS;
        }


        int IndexHandle::load(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont)
        {
            int ret = TFS_SUCCESS;

            if (is_load_)
            {
                return EXIT_INDEX_ALREADY_LOADED_ERROR;
            }

            int64_t file_size = file_op_->get_file_size();
            if (file_size < 0)
            {
                return file_size;
            }
            else if (file_size == 0) /* empty file */
            {
                return EXIT_INDEX_CORRUPT_ERROR;
            }

            MMapOption tmp_map_option = map_option;

            if (file_size > tmp_map_option.first_mmap_size_ && file_size <= tmp_map_option.max_mmap_size_)
            {
                tmp_map_option.first_mmap_size_ = file_size;
            }

            ret = file_op_->mmap_file(tmp_map_option);

            if (ret != TFS_SUCCESS)
            {
                return ret;
            }

            if (0 == bucket_size() || 0 == block_info()->block_id_)
            {
                fprintf(stderr, "lndex corrupt error. blockid: %u, bucket size: %d\n", block_info()->block_id_, bucket_size());
                return EXIT_INDEX_CORRUPT_ERROR;
            }

            /* check file size */
            int32_t index_file_size = sizeof(IndexHandle) + bucket_size() * sizeof(int32_t);

            if (file_size < index_file_size)
            {
                fprintf(stderr, "Index corrupt error. blockid: %u, buck size: %d, file size: %d, index file size: %d\n", block_info()->block_id_, bucket_size(), file_size, index_file_size);
                return EXIT_INDEX_CORRUPT_ERROR;
            }

            /* check block_id_ size */
            if (logic_block_id != block_info()->block_id_)
            {
                fprintf(stderr, "block id conflict. blockid: %u, index blockid: %u\n", logic_block_id, block_info()->block_id_);
                return EXIT_BLOCKID_CONFLICT_ERROR;
            }

            /* check bucket_size */
            if (bucket_size != bucket_size())
            {
                fprintf(stderr, "Index configure error, old bucket size: %d, new bucket size: %d\n", bucket_size(), bucket_size);
                return EXIT_BUCKET_CONFIGURE_ERROR;
            }

            is_load_ = true;

            if (debug)
            {
                printf("load blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, filecount: %d, det_size: %d, del_file_count: %d, version: %d\n", 
                logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
                index_header()->bucket_size_, index_header()->free_head_offset_(), block_info()->seq_no_, block_info()->size_,
                block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
            }

            return TFS_SUCCESS;
        }

    }
}

單元測試 - 索引初始化實現

#include "common.h"
#include "file_op.h"
#include "index_handle.h"
#include "sstream"

using namespace std;
using namespace qiniu;

const static largefile::MMapOption mmap_option = {1024000, 4096, 4096}; /* 內存映射的參數 */
const static uint32_t main_blocksize = 1024 * 1024 * 64; /* 主塊文件的大小 */
const static uint32_t bucket_size = 1000; /* 哈希桶的大小 */
const int32_t block_id = 1;
static int debug = 1;

int main(int argc, char *argv[])
{
    string mainblock_path;
    string index_path;
    int32_t ret = TFS_SUCCESS;

    cout<<"Type your bockid :"<<endl;
    cin >> block_id;

    if (block_id < 1)
    {
        cerr << "Invalid bockid, exit." <<endl;
        exit(-1);
    }


    /* 1. 生成主塊文件 */
    stringstream tmp_stream;
    tmp_stream << "." << MAINBLOCK_DIR_PREFIX << block_id;
    tmp_stream >> mainblock_path;

    largefile::FileOperation *mainblock = new largefile::FileOperation(mainblock_path, O_RDWR | O_LARGEFILE | O_CREATE);

    ret = mainblock->ftruncate_file(main_blocksize);

    if (ret != 0)
    {
        fprintf(stderr, "create main block %s failed. reason: %s\n", mainblock_path, strerror(errno));
        delete mainblock;
        exit(-2);
    }


    /* 2. 創建索引文件 */
    largefile::IndexHandle *index_handle = new largefile::IndexHandle(".", block_id);

    if (debug)
    {
        printf("Init index ...\n");
    }

    ret = index_handle->create(block_id_, bucket_size_, mmap_option);

    if (ret != largefile::TFS_SUCCESS)
    {
        fprintf(stderr, "create index %d failed.\n", block_id);
        delete mainblock;
        delete index_handle;
        exit(-3);
    }

    /* 其他操作 */
    return 0;
}

索引初始化 - 單元測試 & Bug 修復

#ifndef QINIU_LARGEFILE_INDEX_HANDLE_H_
#define QINIU_LARGEFILE_INDEX_HANDLE_H_

#include "common.h"
#include "mmap_file_op.h"

namespace qiniu
{
    namespace largefile
    {
        struct IndexHeader
        {
            public:
            IndexHeader()
            {
                memset(this, 0, sizeof(IndexHeader));
            }

            Blockinfo block_info_; /* meta block info */
            int32_t bucket_size_; /* hash bucket size */
            int32_t data_file_offset_; /* offset to write next data in block */
            int32_t index_file_size_; /* offset after index_header + all bucketa */
            int32_t free_head_offset_; /* free meta node list, for reuse */
        };


        class IndexHandle
        {
            public:
            IndexHandle(const std::string &base_path, const uint32_t main_block_id);
            ~IndexHandle();

            int create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont);
            int load(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont);

            IndexHeader *index_header()
            {
                return reinterpret_cast<IndexHeader *>(file_op_->get_map_data());
            }

            Blockinfo *block_info()
            {
                return reinterpret_cast<Blockinfo *>(file_op_->get_map_data());
            }

            int32_t bucket_size() const
            {
                //return reinterpret_cast<IndexHandle *>(file_op_->get_map_data())->bucket_size;
                reinterpret_cast<IndexHeader *>(file_op_->get_map_data())->bucket_size_;
            }
            
            private:
            MMapFileOperation *file_op_;
            bool is_load_;
        };
        
    }
}
#endif /* QINIU_LARGEFILE_INDEX_HANDLE_H_ */

#include "common.h"
#include "index_handle.h"
#include "sstream"

static int debug = 1;

namespace qiniu
{
    namespace largefile
    {
        /* IndexHandle(const std::string &base_path, const uint32_t main_block_id); */
        IndexHandle::IndexHandle(const std::string &base_path, const uint32_t main_block_id)
        {
            /* create file_op_ handle object */
            std::stringstream tmp_stream;
            tmp_stream << base_path << INDEX_DIR_PREFIX << main_block_id; /* /root/weifc/index/ */

            std::string index_path;
            tmp_stream >> index_path;

            file_op_ = new MMapFileOperation(index_path, O_CREAT | O_RDWR | O_LARGEFILE);
            is_load_ = false;
        }


        IndexHandle::~IndexHandle()
        {
            if (file_op_)
            {
                delete file_op_;
                file_op_ = NULL;
            }
        }


        int IndexHandle::create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont)
        {
            int ret = TFS_SUCCESS;

            if (debug)
            {
                printf("create index, block id: %u, bucket size: %d, max_mmap_size: %d, first mmap size: %d, per mmap size: %d\n",
                        logic_block_id, bucket_size, map_optiont.max_mmap_size_, map_optiont.first_mmap_size_, map_optiont.per_mmap_size_);
            }

            if (is_load_)
            {
                return EXIT_INDEX_ALREADY_LOADED_ERROR;
            }

            int64_t file_size = file_op_->get_file_size();

            if (file_size < 0)
            {
                return TFS_ERROR;
            }else if(file_size == 0) /* empty file */
            {
                IndexHeader i_header;
                i_header.block_info_.block_id_ = logic_block_id;
                i_header.block_info_.seq_no_ = 1;
                i_header.bucket_size_ = bucket_size;

                i_header.index_file_size_ = sizeof(IndexHandle) + bucket_size * sizeof(int32_t);

                /* index header + total buckets */
                char *init_data = new char[i_header.index_file_size_];
                memcpy(init_data, &i_header, sizeof(IndexHandle));
                memset(init_data + sizeof(IndexHandle), 0, i_header.index_file_size_ - sizeof(IndexHandle));

                /* write index header and buckets into file */
                ret = file_op_->pwrite_file(init_data, i_header.index_file_size_, 0);

                delete []init_data;
                init_data = NULL;

                if (ret != TFS_ERROR)
                {
                    return ret;
                }

                ret = file_op_->flush_file();
                if (ret != TFS_SUCCESS)
                {
                    return ret;
                }
            }else /* file size > 0, index already exist */
            {
                return EXIT_META_UNEXPECT_FOUND_ERROR;
            }
            
            ret = file_op_->mmap_file(map_optiont);
            if (ret != TFS_SUCCESS)
            {
                return ret;
            }
            
            is_load_ = true;

            if (debug)
            {
                printf("init blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n", 
                logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
                index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
                block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
            }

            return TFS_SUCCESS;
        }


        int IndexHandle::load(const uint32_t logic_block_id, const int32_t _bucket_size, const MMapOption map_option)
        {
            int ret = TFS_SUCCESS;

            if (is_load_)
            {
                return EXIT_INDEX_ALREADY_LOADED_ERROR;
            }

            int64_t file_size = file_op_->get_file_size();
            if (file_size < 0)
            {
                return file_size;
            }
            else if (file_size == 0) /* empty file */
            {
                return EXIT_INDEX_CORRUPT_ERROR;
            }

            MMapOption tmp_map_option = map_option;

            if (file_size > tmp_map_option.first_mmap_size_ && file_size <= tmp_map_option.max_mmap_size_)
            {
                tmp_map_option.first_mmap_size_ = file_size;
            }

            ret = file_op_->mmap_file(tmp_map_option);

            if (ret != TFS_SUCCESS)
            {
                return ret;
            }

            if (0 == bucket_size() || 0 == block_info()->block_id_)
            {
                fprintf(stderr, "lndex corrupt error. blockid: %u, bucket size: %d\n", block_info()->block_id_, bucket_size());
                return EXIT_INDEX_CORRUPT_ERROR;
            }

            /* check file size */
            int32_t index_file_size = sizeof(IndexHandle) + bucket_size() * sizeof(int32_t);

            if (file_size < index_file_size)
            {
                fprintf(stderr, "Index corrupt error. blockid: %u, buck size: %d, file size: %ld, index file size: %d\n", block_info()->block_id_, bucket_size(), file_size, index_file_size);
                return EXIT_INDEX_CORRUPT_ERROR;
            }

            /* check block_id_ size */
            if (logic_block_id != block_info()->block_id_)
            {
                fprintf(stderr, "block id conflict. blockid: %u, index blockid: %u\n", logic_block_id, block_info()->block_id_);
                return EXIT_BLOCKID_CONFLICT_ERROR;
            }

            /* check bucket_size */
            if (_bucket_size != bucket_size())
            {
                fprintf(stderr, "Index configure error, old bucket size: %d, new bucket size: %d\n", bucket_size(), _bucket_size);
                return EXIT_BUCKET_CONFIGURE_ERROR;
            }

            is_load_ = true;

            if (debug)
            {
                printf("load blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n", 
                logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
                index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
                block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
            }

            return TFS_SUCCESS;
        }

    }
}

#include "common.h"
#include "file_op.h"
#include "index_handle.h"
#include "sstream"

using namespace std;
using namespace qiniu;

const static largefile::MMapOption mmap_option = {1024000, 4096, 4096}; /* 內存映射的參數 */
const static uint32_t main_blocksize = 1024 * 1024 * 64; /* 主塊文件的大小 */
const static uint32_t bucket_size = 1000; /* 哈希桶的大小 */
static int32_t block_id = 1;
static int debug = 1;

int main(int argc, char *argv[])
{
    string mainblock_path;
    string index_path;
    int32_t ret = largefile::TFS_SUCCESS;

    cout << "Type your bockid :" <<endl;
    cin >> block_id;

    if (block_id < 1)
    {
        cerr << "Invalid bockid, exit." <<endl;
        exit(-1);
    }


    /* 1. 生成主塊文件 */
    stringstream tmp_stream;
    tmp_stream << "." << largefile::MAINBLOCK_DIR_PREFIX << block_id;
    tmp_stream >> mainblock_path;

    largefile::FileOperation *mainblock = new largefile::FileOperation(mainblock_path, O_RDWR | O_LARGEFILE | O_CREAT);

    ret = mainblock->ftruncate_file(main_blocksize);

    if (ret != 0)
    {
        fprintf(stderr, "create main block %s failed. reason: %s\n", mainblock_path.c_str(), strerror(errno));
        delete mainblock;
        exit(-2);
    }


    /* 2. 創建索引文件 */
    largefile::IndexHandle *index_handle = new largefile::IndexHandle(".", block_id);

    if (debug)
    {
        printf("Init index ...\n");
    }

    ret = index_handle->create(block_id, bucket_size, mmap_option);

    if (ret != largefile::TFS_SUCCESS)
    {
        fprintf(stderr, "create index %d failed.\n", block_id);
        delete mainblock;
        delete index_handle;
        exit(-3);
    }

    /* 其他操作 */
    return 0;
}

編譯了, 調試了一天的感覺, 終於不完美的成功了. 代碼是看着老師抄的, 思想也是別人的, (只是感覺學了一些新東西, (學習調試報錯)).

編譯:

g++ block_init_test.cpp file_op.cpp index_handle.cpp mmap_file.cpp  mmap_file_op.cpp -o block_init_test

執行:

結語:

調試調了好久! ! !

代碼是跟着老師抄的, 思想也是老師的, 還是好好複習. 可能代碼寫多才有感覺吧? 誰知道呢? 總之, 社會推着你必須學(學習就對了).

時間: 2020-06-27