fastdfs之upload源碼解析

1. upload簡介
fastdfs集羣上傳文件，由client發起。client先連接tracker，獲取目標storage和存儲相關信息；連接目標storage，上傳文件。
2. client upload源代碼解析
2.1 upload_file函數
根據client upload流程提煉出的代碼爲：
int upload_file(const char* local_filename)
{
int result;
ConnectionInfo *pTrackerServer;
ConnectionInfo *pStorageServer;
ConnectionInfo storageServer;
int store_path_index = 0;
char group_name[FDFS_GROUP_NAME_MAX_LEN + 1];
char remote_filename[256];
FDFSMetaData meta_list[32];
int meta_count;
const char *file_ext_name;

/* 連接tracker [client/tracker_client.c]
根據配置文件的tracker服務組：g_tracker_group，和輪詢tracker服務指向標誌：server_index，嘗試連接tracker。如果連接成功，返回連接上的tracker服務；如果不成功，自增server_index，嘗試連接。如果輪詢全部，連接不上，返回NULL。
*/
pTrackerServer = tracker_get_connection();
if (pTrackerServer == NULL)
{
return errno != 0 ? errno : ECONNREFUSED;
}

*group_name = '\0';

/* 查詢目標storage [client/tracker_client.c]
發送命令：TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE，從tracker獲取storage ip地址和端口號，組名，掛載點指向標誌。
*/
if ((result=tracker_query_storage_store(pTrackerServer, \
&storageServer, group_name, &store_path_index)) != 0)
{
/* 斷開tracker服務連接 */
tracker_disconnect_server_ex(pTrackerServer, true);
return result;
}

printf("group_name=%s, ip_addr=%s, port=%d\n", \
group_name, storageServer.ip_addr, \
storageServer.port);

/* 連接storage [tracker/tracker_proto.c]
根據storage的ip_addr和port，client用socket無阻塞模式連接storage。
*/
if ((pStorageServer=tracker_connect_server(&storageServer, \
&result)) == NULL)
{
/* 斷開tracker服務連接 */
tracker_disconnect_server_ex(pTrackerServer, true);
return result;
}

// 需上傳文件的文件屬性
memset(&meta_list, 0, sizeof(meta_list));
meta_count = 0;
strcpy(meta_list[meta_count].name, "ext_name");
strcpy(meta_list[meta_count].value, "jpg");
meta_count++;
strcpy(meta_list[meta_count].name, "width");
strcpy(meta_list[meta_count].value, "160");
meta_count++;
strcpy(meta_list[meta_count].name, "height");
strcpy(meta_list[meta_count].value, "80");
meta_count++;
strcpy(meta_list[meta_count].name, "file_size");
strcpy(meta_list[meta_count].value, "115120");
meta_count++;

// 獲取文件後綴
file_ext_name = fdfs_get_file_ext_name(local_filename);
*group_name = '\0';

/* 上傳文件 [client/storage_client.c]
上傳文件有三個模式：文件fd上傳、緩存buf上傳，回調callback上傳。
這裏用了文件fd上傳，其特點是利用sendfile函數上傳文件。先發送包頭，其中命令爲：STORAGE_PROTO_CMD_UPLOAD_FILE，再用sendfile發送文件內容。
*/
result = storage_upload_by_filename(pTrackerServer, \
pStorageServer, store_path_index, \
local_filename, file_ext_name, \
meta_list, meta_count, \
group_name, remote_filename);

printf("storage_upload_by_filename\n");

if (result != 0)
{
printf("upload file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
}

// 斷開連接
tracker_disconnect_server_ex(pStorageServer, true);
tracker_disconnect_server_ex(pTrackerServer, true);
return result;
}
3. tracker upload相關源碼解析
upload文件，需先連接tracker，再查詢storage。其中連接是純socket連接，不涉及業務，這裏不做分析；查詢storage：tracker_query_storage_store，發送了命令：TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE。
tracker在nio線程[tracker/tracker_nio.c]接收函數client_sock_read接收完後，調用tracker_deal_task [tracker/tracker_service.c]處理。
根據tracker_deal_task其中代碼：
case TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE:
result = tracker_deal_service_query_storage( \
pTask, pHeader->cmd);
break;
其調用了tracker_deal_service_query_storage [tracker/tracker_service.c]函數。
3.1 tracker_deal_service_query_storage函數
tracker/tracker_service.c
static int tracker_deal_service_query_storage( \
struct fast_task_info *pTask, char cmd)
{
int expect_pkg_len;
FDFSGroupInfo *pStoreGroup;
FDFSGroupInfo **ppFoundGroup;
FDFSGroupInfo **ppGroup;
FDFSStorageDetail *pStorageServer;
char *group_name;
char *p;
bool bHaveActiveServer;
int write_path_index;
int avg_reserved_mb;

/* 1. 判斷接收包長
如果是攜帶組名的，應另加組名長度；如果是不攜帶組名的，另加應爲0.
*/
if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE
|| cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ALL)
{
expect_pkg_len = FDFS_GROUP_NAME_MAX_LEN;
}
else
{
expect_pkg_len = 0;
}

if (pTask->length - sizeof(TrackerHeader) != expect_pkg_len)
{
logError("file: "__FILE__", line: %d, " \
"cmd=%d, client ip: %s, package size " \
PKG_LEN_PRINTF_FORMAT"is not correct, " \
"expect length: %d", __LINE__, \
cmd, pTask->client_ip, \
pTask->length - (int)sizeof(TrackerHeader), \
expect_pkg_len);
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}

// 2. 獲取storage組
// 如果不存在storage組，返回ENOENT。
if (g_groups.count == 0)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}

// 攜帶組名的請求
if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE
|| cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ALL)
{
// 在接收到網絡包中提取組名。
group_name = pTask->data + sizeof(TrackerHeader);
group_name[FDFS_GROUP_NAME_MAX_LEN] = '\0';

// 根據組名，獲取組內的storage列表。
pStoreGroup = tracker_mem_get_group(group_name);
if (pStoreGroup == NULL)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, invalid group name: %s", \
__LINE__, pTask->client_ip, group_name);
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}

// 判斷storage組是否存在活動active的storage。
if (pStoreGroup->active_count == 0)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}

// 判斷存儲空間
if (!tracker_check_reserved_space(pStoreGroup))
{
// 存儲空間不足，不使用“合併存儲”，返回ENOSPC，磁盤空間不足。
if (!(g_if_use_trunk_file && \
tracker_check_reserved_space_trunk(pStoreGroup)))
{
pTask->length = sizeof(TrackerHeader);
return ENOSPC;
}
}
} 
// tracker配置的查找storage組策略爲：輪詢和負載均衡
else if (g_groups.store_lookup == FDFS_STORE_LOOKUP_ROUND_ROBIN
||g_groups.store_lookup==FDFS_STORE_LOOKUP_LOAD_BALANCE)
{

int write_group_index;

bHaveActiveServer = false;
// 獲取接下用的組標誌
write_group_index = g_groups.current_write_group;
if (write_group_index >= g_groups.count)
{
write_group_index = 0;
}

pStoreGroup = NULL;
// 如果這一組storage組符合要求，就選定這一組。
// 要求：存在active的storage並存儲空間充足。
ppFoundGroup = g_groups.sorted_groups + write_group_index;
if ((*ppFoundGroup)->active_count > 0)
{
bHaveActiveServer = true;
if (tracker_check_reserved_space(*ppFoundGroup))
{
pStoreGroup = *ppFoundGroup;
}
else if (g_if_use_trunk_file && \
g_groups.store_lookup == \
FDFS_STORE_LOOKUP_LOAD_BALANCE && \
tracker_check_reserved_space_trunk( \
*ppFoundGroup))
{
pStoreGroup = *ppFoundGroup;
}
}

// 如果指定的storage組不符合要求，自增輪詢，查找符合要求的組。
if (pStoreGroup == NULL)
{
// 先從write_group_index的位置往後找
FDFSGroupInfo **ppGroupEnd;
ppGroupEnd = g_groups.sorted_groups + \
     g_groups.count;
for (ppGroup=ppFoundGroup+1; \
ppGroup<ppGroupEnd; ppGroup++)
{
if ((*ppGroup)->active_count == 0)
{
continue;
}

bHaveActiveServer = true;
if (tracker_check_reserved_space(*ppGroup))
{
pStoreGroup = *ppGroup;
g_groups.current_write_group = \
       ppGroup-g_groups.sorted_groups;
break;
}
}

// 從開始到write_group_index的位置找
if (pStoreGroup == NULL)
{
for (ppGroup=g_groups.sorted_groups; \
ppGroup<ppFoundGroup; ppGroup++)
{
if ((*ppGroup)->active_count == 0)
{
continue;
}

bHaveActiveServer = true;
if (tracker_check_reserved_space(*ppGroup))
{
pStoreGroup = *ppGroup;
g_groups.current_write_group = \
ppGroup-g_groups.sorted_groups;
break;
}
}
}
// 如果沒找到
if (pStoreGroup == NULL)
{
// 不存在active的storage，返回ENOENT，不存在。
if (!bHaveActiveServer)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}

// 存在active的storage，但不使用“合併存儲”，返回ENOSPC，空間不足。
if (!g_if_use_trunk_file)
{
pTask->length = sizeof(TrackerHeader);
return ENOSPC;
}

// 以下爲使用“合併存儲”情況
for (ppGroup=g_groups.sorted_groups; \
ppGroup<ppGroupEnd; ppGroup++)
{
if ((*ppGroup)->active_count == 0)
{
continue;
}
if (tracker_check_reserved_space_trunk(*ppGroup))
{
pStoreGroup = *ppGroup;
g_groups.current_write_group = \
ppGroup-g_groups.sorted_groups;
break;
}
}

if (pStoreGroup == NULL)
{
pTask->length = sizeof(TrackerHeader);
return ENOSPC;
}
}
}

// 找到合適的storage組，如果是輪詢模式，自增下次獲取的標誌。
if (g_groups.store_lookup == FDFS_STORE_LOOKUP_ROUND_ROBIN)
{
g_groups.current_write_group++;
if (g_groups.current_write_group >= g_groups.count)
{
g_groups.current_write_group = 0;
}
}
} 
// 配置爲指定storage組模式。如果指定組符合要求，則使用該組；如果不符合，返回EINVAL，無效
// 要求：存在active的storage並存儲空間充足。
else if (g_groups.store_lookup == FDFS_STORE_LOOKUP_SPEC_GROUP)
{
if (g_groups.pStoreGroup == NULL || \
g_groups.pStoreGroup->active_count == 0)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}

if (!tracker_check_reserved_space(g_groups.pStoreGroup))
{
if (!(g_if_use_trunk_file && \
tracker_check_reserved_space_trunk( \
g_groups.pStoreGroup)))
{
pTask->length = sizeof(TrackerHeader);
return ENOSPC;
}
}

pStoreGroup = g_groups.pStoreGroup;
}
else
{
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}

// storage組無掛載點，返回ENOENT，不存在
if (pStoreGroup->store_path_count <= 0)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}

// 3. 從storage組內獲取storage
if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE
  || cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE)
{
pStorageServer = tracker_get_writable_storage(pStoreGroup);
if (pStorageServer == NULL)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}
}
else  //query store server list, use the first to check
{
pStorageServer = *(pStoreGroup->active_servers);
}

// 4. 獲取存儲的掛載點store_path
write_path_index = pStorageServer->current_write_path;
if (write_path_index >= pStoreGroup->store_path_count)
{
write_path_index = 0;
}


// 判斷storage的store_path剩餘空間，獲取充足空間的store_path。
avg_reserved_mb = g_storage_reserved_space.rs.mb / \
  pStoreGroup->store_path_count;
if (!tracker_check_reserved_space_path(pStorageServer-> \
path_total_mbs[write_path_index], pStorageServer-> \
path_free_mbs[write_path_index], avg_reserved_mb))
{
int i;
for (i=0; i<pStoreGroup->store_path_count; i++)
{
if (tracker_check_reserved_space_path( \
pStorageServer->path_total_mbs[i], \
pStorageServer->path_free_mbs[i], \
avg_reserved_mb))
{
pStorageServer->current_write_path = i;
write_path_index = i;
break;
}
}

if (i == pStoreGroup->store_path_count)
{
if (!g_if_use_trunk_file)
{
pTask->length = sizeof(TrackerHeader);
return ENOSPC;
}

for (i=write_path_index; i<pStoreGroup-> \
store_path_count; i++)
{
if (tracker_check_reserved_space_path( \
  pStorageServer->path_total_mbs[i], \
  pStorageServer->path_free_mbs[i] + \
  pStoreGroup->trunk_free_mb, \
  avg_reserved_mb))
{
pStorageServer->current_write_path = i;
write_path_index = i;
break;
}
}
if ( i == pStoreGroup->store_path_count)
{
for (i=0; i<write_path_index; i++)
{
      if (tracker_check_reserved_space_path( \
              pStorageServer->path_total_mbs[i], \
              pStorageServer->path_free_mbs[i] + \
              pStoreGroup->trunk_free_mb, \
              avg_reserved_mb))
    {
pStorageServer->current_write_path = i;
write_path_index = i;
break;
    }
}

if (i == write_path_index)
{
pTask->length = sizeof(TrackerHeader);
return ENOSPC;
}
}
}
}

// 獲取到store_path後，更新下次獲取的位置
if (g_groups.store_path == FDFS_STORE_PATH_ROUND_ROBIN)
{
pStorageServer->current_write_path++;
if (pStorageServer->current_write_path >= \
pStoreGroup->store_path_count)
{
pStorageServer->current_write_path = 0;
}
}


/*
//printf("pStoreGroup->current_write_server: %d, " \
"pStoreGroup->active_count=%d\n", \
pStoreGroup->current_write_server, \
pStoreGroup->active_count);
*/
/* 獲取到storage和store_path後，填寫返回信息
包括：組名，storage信息，store_path標誌。
*/

        // 5. 組裝返回包信息

p = pTask->data + sizeof(TrackerHeader);

        // 存儲組名

memcpy(p, pStoreGroup->group_name, FDFS_GROUP_NAME_MAX_LEN);
p += FDFS_GROUP_NAME_MAX_LEN;

// 如果是獲取整組的storage，則填寫整組active的storage信息，否則僅填寫獲取到的storage。
if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ALL
|| cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ALL)
{
int active_count;
FDFSStorageDetail **ppServer;
FDFSStorageDetail **ppEnd;

active_count = pStoreGroup->active_count;
if (active_count == 0)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}

ppEnd = pStoreGroup->active_servers + active_count;
for (ppServer=pStoreGroup->active_servers; ppServer<ppEnd; \
ppServer++)
{
memcpy(p, (*ppServer)->ip_addr, IP_ADDRESS_SIZE - 1);
p += IP_ADDRESS_SIZE - 1;

long2buff(pStoreGroup->storage_port, p);
p += FDFS_PROTO_PKG_LEN_SIZE;
}
}
else
{
memcpy(p, pStorageServer->ip_addr, IP_ADDRESS_SIZE - 1);
p += IP_ADDRESS_SIZE - 1;

long2buff(pStoreGroup->storage_port, p);
p += FDFS_PROTO_PKG_LEN_SIZE;
}

*p++ = (char)write_path_index;
pTask->length = p - pTask->data;

return 0;
}

根據源代碼解析，最終選擇到的storage是active storage，其存儲空間充足。選中的store_path也是空間充足的路徑。

4. storage upload相關源碼解析
upload文件在與tracker通信獲取storage後，先與storage連接，再上傳文件。其中與storage連接爲socket連接；上傳文件（文件上傳方式），先發送包頭，發送了命令：STORAGE_PROTO_CMD_UPLOAD_FILE，再使用sendfile函數上傳文件。
在storage端，由accept線程接收連接，通知nio網絡線程接收數據。nio線程解析處理後，由dio磁盤線程寫入文件。
accept線接收連接，nio線程接收數據等屬於storage框架內容，這裏不深入分析。我們從nio線程解析處理代碼開始分析。
nio線程解析處理函數爲storage_deal_task[storage/storage_service.c]，其中與upload相關的爲：
case STORAGE_PROTO_CMD_UPLOAD_FILE:
ACCESS_LOG_INIT_FIELDS();
result = storage_upload_file(pTask, false);
STORAGE_ACCESS_LOG(pTask, \
ACCESS_LOG_ACTION_UPLOAD_FILE, \
result);
break;
其調用了storage_upload_file[storage/storage_service.c]函數。
4.1 storage_upload_file函數
storage/storage_service.c
/**
1 byte: store path index
8 bytes: file size 
FDFS_FILE_EXT_NAME_MAX_LEN bytes: file ext name, do not include dot (.)
file size bytes: file content
**/
static int storage_upload_file(struct fast_task_info *pTask, bool bAppenderFile)
{
StorageClientInfo *pClientInfo;
StorageFileContext *pFileContext;
DisconnectCleanFunc clean_func;
char *p;
char filename[128];
char file_ext_name[FDFS_FILE_PREFIX_MAX_LEN + 1];
int64_t nInPackLen;
int64_t file_offset;
int64_t file_bytes;
int crc32;
int store_path_index;
int result;
int filename_len;

// 1. 獲取連接信息，判斷包長是否正確。
pClientInfo = (StorageClientInfo *)pTask->arg;
pFileContext =  &(pClientInfo->file_context);
nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader);

if (nInPackLen < 1 + FDFS_PROTO_PKG_LEN_SIZE + 
FDFS_FILE_EXT_NAME_MAX_LEN)
{
logError("file: "__FILE__", line: %d, " \
"cmd=%d, client ip: %s, package size " \
"%"PRId64" is not correct, " \
"expect length >= %d", __LINE__, \
STORAGE_PROTO_CMD_UPLOAD_FILE, \
pTask->client_ip,  nInPackLen, \
1 + FDFS_PROTO_PKG_LEN_SIZE + \
FDFS_FILE_EXT_NAME_MAX_LEN);
return EINVAL;
}

// 2. 獲取store_path。
/* 如果爲-1，重新獲取store_path_index；如果小於0或者大於store_path個數，返回錯誤*/
p = pTask->data + sizeof(TrackerHeader);
store_path_index = *p++;

if (store_path_index == -1)
{
// 獲取存儲充足的store_path
if ((result=storage_get_storage_path_index( \
&store_path_index)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"get_storage_path_index fail, " \
"errno: %d, error info: %s", __LINE__, \
result, STRERROR(result));
return result;
}
}
else if (store_path_index < 0 || store_path_index >= \
g_fdfs_store_paths.count)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, store_path_index: %d " \
"is invalid", __LINE__, \
pTask->client_ip, store_path_index);
return EINVAL;
}

// 3.獲取文件大小
file_bytes = buff2long(p);
p += FDFS_PROTO_PKG_LEN_SIZE;
if (file_bytes < 0 || file_bytes != nInPackLen - \
(1 + FDFS_PROTO_PKG_LEN_SIZE + \
FDFS_FILE_EXT_NAME_MAX_LEN))
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, pkg length is not correct, " \
"invalid file bytes: %"PRId64 \
", total body length: %"PRId64, \
__LINE__, pTask->client_ip, file_bytes, nInPackLen);
return EINVAL;
}

// 4.獲取文件後綴
memcpy(file_ext_name, p, FDFS_FILE_EXT_NAME_MAX_LEN);
*(file_ext_name + FDFS_FILE_EXT_NAME_MAX_LEN) = '\0';
p += FDFS_FILE_EXT_NAME_MAX_LEN;
if ((result=fdfs_validate_filename(file_ext_name)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, file_ext_name: %s " \
"is invalid!", __LINE__, \
pTask->client_ip, file_ext_name);
return result;
}

// 5.封裝pFileContext
pFileContext->calc_crc32 = true;
pFileContext->calc_file_hash = g_check_file_duplicate;
pFileContext->extra_info.upload.start_time = g_current_time;

strcpy(pFileContext->extra_info.upload.file_ext_name, file_ext_name);
storage_format_ext_name(file_ext_name, \
pFileContext->extra_info.upload.formatted_ext_name);
pFileContext->extra_info.upload.trunk_info.path. \
store_path_index = store_path_index;
pFileContext->extra_info.upload.file_type = _FILE_TYPE_REGULAR;
pFileContext->sync_flag = STORAGE_OP_TYPE_SOURCE_CREATE_FILE;
pFileContext->timestamp2log = pFileContext->extra_info.upload.start_time;
pFileContext->op = FDFS_STORAGE_FILE_OP_WRITE;
if (bAppenderFile)
{
// 分塊上傳
pFileContext->extra_info.upload.file_type |= \
_FILE_TYPE_APPENDER;
}
else
{
// 判斷是否使用“合併存儲”
if (g_if_use_trunk_file && trunk_check_size( \
TRUNK_CALC_SIZE(file_bytes)))
{
pFileContext->extra_info.upload.file_type |= \
_FILE_TYPE_TRUNK;
}
}

// 如果使用“合併存儲”
if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_TRUNK)
{
FDFSTrunkFullInfo *pTrunkInfo;

pFileContext->extra_info.upload.if_sub_path_alloced = true;
pTrunkInfo = &(pFileContext->extra_info.upload.trunk_info);
if ((result=trunk_client_trunk_alloc_space( \
TRUNK_CALC_SIZE(file_bytes), pTrunkInfo)) != 0)
{
return result;
}

clean_func = dio_trunk_write_finish_clean_up;
file_offset = TRUNK_FILE_START_OFFSET((*pTrunkInfo));
pFileContext->extra_info.upload.if_gen_filename = true;
trunk_get_full_filename(pTrunkInfo, pFileContext->filename, \
sizeof(pFileContext->filename));
pFileContext->extra_info.upload.before_open_callback = \
dio_check_trunk_file_when_upload;
pFileContext->extra_info.upload.before_close_callback = \
dio_write_chunk_header;
pFileContext->open_flags = O_RDWR | g_extra_open_file_flags;
}
else // 非“合併存儲”情況
{
char reserved_space_str[32];

// 判斷剩餘空間是否滿足當前文件存儲
if (!storage_check_reserved_space_path(g_path_space_list \
[store_path_index].total_mb, g_path_space_list \
[store_path_index].free_mb - (file_bytes/FDFS_ONE_MB), \
g_avg_storage_reserved_mb))
{
logError("file: "__FILE__", line: %d, " \
"no space to upload file, "
"free space: %d MB is too small, file bytes: " \
"%"PRId64", reserved space: %s", \
__LINE__, g_path_space_list[store_path_index].\
free_mb, file_bytes, \
fdfs_storage_reserved_space_to_string_ex( \
g_storage_reserved_space.flag, \
g_avg_storage_reserved_mb, \
g_path_space_list[store_path_index]. \
total_mb, g_storage_reserved_space.rs.ratio,\
reserved_space_str));
return ENOSPC;
}


// 生成文件名稱
crc32 = rand();
*filename = '\0';
filename_len = 0;
pFileContext->extra_info.upload.if_sub_path_alloced = false;
if ((result=storage_get_filename(pClientInfo, \
pFileContext->extra_info.upload.start_time, \
file_bytes, crc32, pFileContext->extra_info.upload.\
formatted_ext_name, filename, &filename_len, \
pFileContext->filename)) != 0)
{
return result;
}

// dio線程處理函數中用到的清除函數clean_func
// 關閉打開的pFileContext fd，判斷如果未完成寫入，就刪除該文件。
clean_func = dio_write_finish_clean_up;
file_offset = 0;
pFileContext->extra_info.upload.if_gen_filename = true;
pFileContext->extra_info.upload.before_open_callback = NULL;
pFileContext->extra_info.upload.before_close_callback = NULL;
pFileContext->open_flags = O_WRONLY | O_CREAT | O_TRUNC \
| g_extra_open_file_flags;
}

/* 6.繼續封裝pFileContext，並把pTask壓入dio隊列。
包括封裝文件內容，大小，偏移量，任務處理函數, 完成回調函數，清除函數
封裝後，把這個任務壓入dio線程的隊列，再由dio線程讀取隊列，調用任務處理函數處理任務
任務處理函數:dio_write_file 把上傳內容寫入文件，更新文件相關的屬性。如果失敗，調用清除函數。最後調用完成回調函數。
完成回調函數：storage_upload_file_done_callback 封裝pFileContext之中的文件上傳的組名，fileid，記錄binlog等。
清除函數：dio_write_finish_clean_up 上面有介紹
*/
return storage_write_to_file(pTask, file_offset, file_bytes, \
p - pTask->data, dio_write_file, \
storage_upload_file_done_callback, \
clean_func, store_path_index);
}


其中主要的回調函數爲：dio_write_file[storage/storage_dio.c]，storage_upload_file_done_callback[storage_service.c]，dio_write_finish_clean_up[storage/storage_dio.c]。
4.2 dio_write_file函數
在storage_write_to_file函數中，封裝pFileContext，也就封裝了pTask。並且把這個pTask壓入dio線程的隊列中。在dio線程dio_thread_entrance[storage/storage_dio.c]函數中，讀取隊列任務，調用了回調函數，如下：
pContext = (struct storage_dio_context *)arg; 
while (g_continue_flag)
{
while ((pTask=blocked_queue_pop(&(pContext->queue))) != NULL)
{
((StorageClientInfo *)pTask->arg)->deal_func(pTask);
}
}
此時，在這裏的deal_func就是dio_write_file函數。

storage/storage_dio.c
int dio_write_file(struct fast_task_info *pTask)
{
StorageClientInfo *pClientInfo;
StorageFileContext *pFileContext;
int result;
int write_bytes;
char *pDataBuff;

// 1.獲取client信息和文件信息
pClientInfo = (StorageClientInfo *)pTask->arg;
pFileContext = &(pClientInfo->file_context);
result = 0;
do
{
// 2.打開文件寫入的fd
if (pFileContext->fd < 0)
{
// 如果註冊打開前回調函數，即調用處理。在upload過程中，無註冊。
if (pFileContext->extra_info.upload.before_open_callback!=NULL)
{
result = pFileContext->extra_info.upload. \
before_open_callback(pTask);
if (result != 0)
{
break;
}
}
// 根據文件路徑打開文件，並設置偏移量。
if ((result=dio_open_file(pFileContext)) != 0)
{
break;
}
}

// 3.寫入文件內容
pDataBuff = pTask->data + pFileContext->buff_offset;
write_bytes = pTask->length - pFileContext->buff_offset;
if (fc_safe_write(pFileContext->fd, pDataBuff, write_bytes) != write_bytes)
{
result = errno != 0 ? errno : EIO;
logError("file: "__FILE__", line: %d, " \
"write to file: %s fail, fd=%d, write_bytes=%d, " \
"errno: %d, error info: %s", \
__LINE__, pFileContext->filename, \
pFileContext->fd, write_bytes, \
result, STRERROR(result));
}

// 4.更新storage status
pthread_mutex_lock(&g_dio_thread_lock);
g_storage_stat.total_file_write_count++;
if (result == 0)
{
g_storage_stat.success_file_write_count++;
}
pthread_mutex_unlock(&g_dio_thread_lock);


if (result != 0)
{
break;
}


// 5.文件校驗信息
if (pFileContext->calc_crc32)
{
pFileContext->crc32 = CRC32_ex(pDataBuff, write_bytes, \
pFileContext->crc32);
}

if (pFileContext->calc_file_hash)
{
if (g_file_signature_method == STORAGE_FILE_SIGNATURE_METHOD_HASH)
{
CALC_HASH_CODES4(pDataBuff, write_bytes, \
pFileContext->file_hash_codes)
}
else
{
my_md5_update(&pFileContext->md5_context, \
(unsigned char *)pDataBuff, write_bytes);
}
}

/*
logInfo("###dio write bytes: %d, pTask->length=%d, buff_offset=%d", \
write_bytes, pTask->length, pFileContext->buff_offset);
*/

pFileContext->offset += write_bytes;
if (pFileContext->offset < pFileContext->end)
{
// 文件寫入不完整，通知nio線程處理【繼續讀取信息】
pFileContext->buff_offset = 0;
storage_nio_notify(pTask);  //notify nio to deal
}
else
{
// 文件寫入成功，封裝校驗信息
if (pFileContext->calc_crc32)
{
pFileContext->crc32 = CRC32_FINAL( \
pFileContext->crc32);
}

if (pFileContext->calc_file_hash)
{
if (g_file_signature_method == STORAGE_FILE_SIGNATURE_METHOD_HASH)
{
FINISH_HASH_CODES4(pFileContext->file_hash_codes)
}
else
{
my_md5_final((unsigned char *)(pFileContext-> \
file_hash_codes), &pFileContext->md5_context);
}
}

// 如果註冊了關閉前回調函數，即調用處理。在upload過程中，無註冊。
if (pFileContext->extra_info.upload.before_close_callback != NULL)
{
result = pFileContext->extra_info.upload. \
before_close_callback(pTask);
}

// 關閉打開的fd
/* file write done, close it */
close(pFileContext->fd);
pFileContext->fd = -1;

// 如果註冊完成回調函數，即調用處理。在upload過程中，註冊storage_upload_file_done_callback。
if (pFileContext->done_callback != NULL)
{
pFileContext->done_callback(pTask, result);
}
}

return 0;
} while (0);


// 失敗返回。調用清除函數。在upload過程中註冊了dio_write_finish_clean_up。
pClientInfo->clean_func(pTask);
// 如果註冊完成回調函數，即調用處理。
if (pFileContext->done_callback != NULL)
{
pFileContext->done_callback(pTask, result);
}
return result;
}
4.3 storage_upload_file_done_callback函數
在upload的過程中，dio_write_file中任務完成後調用done_callback ，在這裏done_callback就是storage_upload_file_done_callback函數。
storage/storage_service.c
static void storage_upload_file_done_callback(struct fast_task_info *pTask, \
const int err_no)
{
StorageClientInfo *pClientInfo;
StorageFileContext *pFileContext;
TrackerHeader *pHeader;
int result;

// 1.獲取client信息和文件信息
pClientInfo = (StorageClientInfo *)pTask->arg;
pFileContext =  &(pClientInfo->file_context);

// 2.確定返回碼
if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_TRUNK)
{
result = trunk_client_trunk_alloc_confirm( \
&(pFileContext->extra_info.upload.trunk_info), err_no);
if (err_no != 0)
{
result = err_no;
}
}
else
{
result = err_no;
}

// 3.寫binlog，確定需要返回的組名和文件id等
if (result == 0)
{
result = storage_service_upload_file_done(pTask);
if (result == 0)
{
if (pFileContext->create_flag & STORAGE_CREATE_FLAG_FILE)
{
result = storage_binlog_write(\
pFileContext->timestamp2log, \
STORAGE_OP_TYPE_SOURCE_CREATE_FILE, \
pFileContext->fname2log);
}
}
}

// 4.更新storage status。
if (result == 0)
{
int filename_len;
char *p;

if (pFileContext->create_flag & STORAGE_CREATE_FLAG_FILE)
{
CHECK_AND_WRITE_TO_STAT_FILE3_WITH_BYTES( \
g_storage_stat.total_upload_count, \
g_storage_stat.success_upload_count, \
g_storage_stat.last_source_update, \
g_storage_stat.total_upload_bytes, \
g_storage_stat.success_upload_bytes, \
pFileContext->end - pFileContext->start)
}

filename_len = strlen(pFileContext->fname2log);
pClientInfo->total_length = sizeof(TrackerHeader) + \
FDFS_GROUP_NAME_MAX_LEN + filename_len;
p = pTask->data + sizeof(TrackerHeader);
memcpy(p, pFileContext->extra_info.upload.group_name, \
FDFS_GROUP_NAME_MAX_LEN);
p += FDFS_GROUP_NAME_MAX_LEN;
memcpy(p, pFileContext->fname2log, filename_len);
}
else
{
pthread_mutex_lock(&stat_count_thread_lock);
if (pFileContext->create_flag & STORAGE_CREATE_FLAG_FILE)
{
g_storage_stat.total_upload_count++;
  g_storage_stat.total_upload_bytes += \
pClientInfo->total_offset;
}
pthread_mutex_unlock(&stat_count_thread_lock);

pClientInfo->total_length = sizeof(TrackerHeader);
}

STORAGE_ACCESS_LOG(pTask, ACCESS_LOG_ACTION_UPLOAD_FILE, result);

// 5.封裝dio返回給nio的pTask。
pClientInfo->total_offset = 0;
pTask->length = pClientInfo->total_length;

pHeader = (TrackerHeader *)pTask->data;
pHeader->status = result;
pHeader->cmd = STORAGE_PROTO_CMD_RESP;
long2buff(pClientInfo->total_length - sizeof(TrackerHeader), \
pHeader->pkg_len);

        // 6.通知nio線程處理【此時，包已接收完畢，直接網絡發送給client。】
storage_nio_notify(pTask);
}