fastdfs之upload源码解析

1. upload简介
fastdfs集群上传文件，由client发起。client先连接tracker，获取目标storage和存储相关信息；连接目标storage，上传文件。
2. client upload源代码解析
2.1 upload_file函数
根据client upload流程提炼出的代码为：
int upload_file(const char* local_filename)
{
int result;
ConnectionInfo *pTrackerServer;
ConnectionInfo *pStorageServer;
ConnectionInfo storageServer;
int store_path_index = 0;
char group_name[FDFS_GROUP_NAME_MAX_LEN + 1];
char remote_filename[256];
FDFSMetaData meta_list[32];
int meta_count;
const char *file_ext_name;

/* 连接tracker [client/tracker_client.c]
根据配置文件的tracker服务组：g_tracker_group，和轮询tracker服务指向标志：server_index，尝试连接tracker。如果连接成功，返回连接上的tracker服务；如果不成功，自增server_index，尝试连接。如果轮询全部，连接不上，返回NULL。
*/
pTrackerServer = tracker_get_connection();
if (pTrackerServer == NULL)
{
return errno != 0 ? errno : ECONNREFUSED;
}

*group_name = '\0';

/* 查询目标storage [client/tracker_client.c]
发送命令：TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE，从tracker获取storage ip地址和端口号，组名，挂载点指向标志。
*/
if ((result=tracker_query_storage_store(pTrackerServer, \
&storageServer, group_name, &store_path_index)) != 0)
{
/* 断开tracker服务连接 */
tracker_disconnect_server_ex(pTrackerServer, true);
return result;
}

printf("group_name=%s, ip_addr=%s, port=%d\n", \
group_name, storageServer.ip_addr, \
storageServer.port);

/* 连接storage [tracker/tracker_proto.c]
根据storage的ip_addr和port，client用socket无阻塞模式连接storage。
*/
if ((pStorageServer=tracker_connect_server(&storageServer, \
&result)) == NULL)
{
/* 断开tracker服务连接 */
tracker_disconnect_server_ex(pTrackerServer, true);
return result;
}

// 需上传文件的文件属性
memset(&meta_list, 0, sizeof(meta_list));
meta_count = 0;
strcpy(meta_list[meta_count].name, "ext_name");
strcpy(meta_list[meta_count].value, "jpg");
meta_count++;
strcpy(meta_list[meta_count].name, "width");
strcpy(meta_list[meta_count].value, "160");
meta_count++;
strcpy(meta_list[meta_count].name, "height");
strcpy(meta_list[meta_count].value, "80");
meta_count++;
strcpy(meta_list[meta_count].name, "file_size");
strcpy(meta_list[meta_count].value, "115120");
meta_count++;

// 获取文件后缀
file_ext_name = fdfs_get_file_ext_name(local_filename);
*group_name = '\0';

/* 上传文件 [client/storage_client.c]
上传文件有三个模式：文件fd上传、缓存buf上传，回调callback上传。
这里用了文件fd上传，其特点是利用sendfile函数上传文件。先发送包头，其中命令为：STORAGE_PROTO_CMD_UPLOAD_FILE，再用sendfile发送文件内容。
*/
result = storage_upload_by_filename(pTrackerServer, \
pStorageServer, store_path_index, \
local_filename, file_ext_name, \
meta_list, meta_count, \
group_name, remote_filename);

printf("storage_upload_by_filename\n");

if (result != 0)
{
printf("upload file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
}

// 断开连接
tracker_disconnect_server_ex(pStorageServer, true);
tracker_disconnect_server_ex(pTrackerServer, true);
return result;
}
3. tracker upload相关源码解析
upload文件，需先连接tracker，再查询storage。其中连接是纯socket连接，不涉及业务，这里不做分析；查询storage：tracker_query_storage_store，发送了命令：TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE。
tracker在nio线程[tracker/tracker_nio.c]接收函数client_sock_read接收完后，调用tracker_deal_task [tracker/tracker_service.c]处理。
根据tracker_deal_task其中代码：
case TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE:
result = tracker_deal_service_query_storage( \
pTask, pHeader->cmd);
break;
其调用了tracker_deal_service_query_storage [tracker/tracker_service.c]函数。
3.1 tracker_deal_service_query_storage函数
tracker/tracker_service.c
static int tracker_deal_service_query_storage( \
struct fast_task_info *pTask, char cmd)
{
int expect_pkg_len;
FDFSGroupInfo *pStoreGroup;
FDFSGroupInfo **ppFoundGroup;
FDFSGroupInfo **ppGroup;
FDFSStorageDetail *pStorageServer;
char *group_name;
char *p;
bool bHaveActiveServer;
int write_path_index;
int avg_reserved_mb;

/* 1. 判断接收包长
如果是携带组名的，应另加组名长度；如果是不携带组名的，另加应为0.
*/
if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE
|| cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ALL)
{
expect_pkg_len = FDFS_GROUP_NAME_MAX_LEN;
}
else
{
expect_pkg_len = 0;
}

if (pTask->length - sizeof(TrackerHeader) != expect_pkg_len)
{
logError("file: "__FILE__", line: %d, " \
"cmd=%d, client ip: %s, package size " \
PKG_LEN_PRINTF_FORMAT"is not correct, " \
"expect length: %d", __LINE__, \
cmd, pTask->client_ip, \
pTask->length - (int)sizeof(TrackerHeader), \
expect_pkg_len);
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}

// 2. 获取storage组
// 如果不存在storage组，返回ENOENT。
if (g_groups.count == 0)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}

// 携带组名的请求
if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE
|| cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ALL)
{
// 在接收到网络包中提取组名。
group_name = pTask->data + sizeof(TrackerHeader);
group_name[FDFS_GROUP_NAME_MAX_LEN] = '\0';

// 根据组名，获取组内的storage列表。
pStoreGroup = tracker_mem_get_group(group_name);
if (pStoreGroup == NULL)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, invalid group name: %s", \
__LINE__, pTask->client_ip, group_name);
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}

// 判断storage组是否存在活动active的storage。
if (pStoreGroup->active_count == 0)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}

// 判断存储空间
if (!tracker_check_reserved_space(pStoreGroup))
{
// 存储空间不足，不使用“合并存储”，返回ENOSPC，磁盘空间不足。
if (!(g_if_use_trunk_file && \
tracker_check_reserved_space_trunk(pStoreGroup)))
{
pTask->length = sizeof(TrackerHeader);
return ENOSPC;
}
}
} 
// tracker配置的查找storage组策略为：轮询和负载均衡
else if (g_groups.store_lookup == FDFS_STORE_LOOKUP_ROUND_ROBIN
||g_groups.store_lookup==FDFS_STORE_LOOKUP_LOAD_BALANCE)
{

int write_group_index;

bHaveActiveServer = false;
// 获取接下用的组标志
write_group_index = g_groups.current_write_group;
if (write_group_index >= g_groups.count)
{
write_group_index = 0;
}

pStoreGroup = NULL;
// 如果这一组storage组符合要求，就选定这一组。
// 要求：存在active的storage并存储空间充足。
ppFoundGroup = g_groups.sorted_groups + write_group_index;
if ((*ppFoundGroup)->active_count > 0)
{
bHaveActiveServer = true;
if (tracker_check_reserved_space(*ppFoundGroup))
{
pStoreGroup = *ppFoundGroup;
}
else if (g_if_use_trunk_file && \
g_groups.store_lookup == \
FDFS_STORE_LOOKUP_LOAD_BALANCE && \
tracker_check_reserved_space_trunk( \
*ppFoundGroup))
{
pStoreGroup = *ppFoundGroup;
}
}

// 如果指定的storage组不符合要求，自增轮询，查找符合要求的组。
if (pStoreGroup == NULL)
{
// 先从write_group_index的位置往后找
FDFSGroupInfo **ppGroupEnd;
ppGroupEnd = g_groups.sorted_groups + \
     g_groups.count;
for (ppGroup=ppFoundGroup+1; \
ppGroup<ppGroupEnd; ppGroup++)
{
if ((*ppGroup)->active_count == 0)
{
continue;
}

bHaveActiveServer = true;
if (tracker_check_reserved_space(*ppGroup))
{
pStoreGroup = *ppGroup;
g_groups.current_write_group = \
       ppGroup-g_groups.sorted_groups;
break;
}
}

// 从开始到write_group_index的位置找
if (pStoreGroup == NULL)
{
for (ppGroup=g_groups.sorted_groups; \
ppGroup<ppFoundGroup; ppGroup++)
{
if ((*ppGroup)->active_count == 0)
{
continue;
}

bHaveActiveServer = true;
if (tracker_check_reserved_space(*ppGroup))
{
pStoreGroup = *ppGroup;
g_groups.current_write_group = \
ppGroup-g_groups.sorted_groups;
break;
}
}
}
// 如果没找到
if (pStoreGroup == NULL)
{
// 不存在active的storage，返回ENOENT，不存在。
if (!bHaveActiveServer)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}

// 存在active的storage，但不使用“合并存储”，返回ENOSPC，空间不足。
if (!g_if_use_trunk_file)
{
pTask->length = sizeof(TrackerHeader);
return ENOSPC;
}

// 以下为使用“合并存储”情况
for (ppGroup=g_groups.sorted_groups; \
ppGroup<ppGroupEnd; ppGroup++)
{
if ((*ppGroup)->active_count == 0)
{
continue;
}
if (tracker_check_reserved_space_trunk(*ppGroup))
{
pStoreGroup = *ppGroup;
g_groups.current_write_group = \
ppGroup-g_groups.sorted_groups;
break;
}
}

if (pStoreGroup == NULL)
{
pTask->length = sizeof(TrackerHeader);
return ENOSPC;
}
}
}

// 找到合适的storage组，如果是轮询模式，自增下次获取的标志。
if (g_groups.store_lookup == FDFS_STORE_LOOKUP_ROUND_ROBIN)
{
g_groups.current_write_group++;
if (g_groups.current_write_group >= g_groups.count)
{
g_groups.current_write_group = 0;
}
}
} 
// 配置为指定storage组模式。如果指定组符合要求，则使用该组；如果不符合，返回EINVAL，无效
// 要求：存在active的storage并存储空间充足。
else if (g_groups.store_lookup == FDFS_STORE_LOOKUP_SPEC_GROUP)
{
if (g_groups.pStoreGroup == NULL || \
g_groups.pStoreGroup->active_count == 0)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}

if (!tracker_check_reserved_space(g_groups.pStoreGroup))
{
if (!(g_if_use_trunk_file && \
tracker_check_reserved_space_trunk( \
g_groups.pStoreGroup)))
{
pTask->length = sizeof(TrackerHeader);
return ENOSPC;
}
}

pStoreGroup = g_groups.pStoreGroup;
}
else
{
pTask->length = sizeof(TrackerHeader);
return EINVAL;
}

// storage组无挂载点，返回ENOENT，不存在
if (pStoreGroup->store_path_count <= 0)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}

// 3. 从storage组内获取storage
if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE
  || cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE)
{
pStorageServer = tracker_get_writable_storage(pStoreGroup);
if (pStorageServer == NULL)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}
}
else  //query store server list, use the first to check
{
pStorageServer = *(pStoreGroup->active_servers);
}

// 4. 获取存储的挂载点store_path
write_path_index = pStorageServer->current_write_path;
if (write_path_index >= pStoreGroup->store_path_count)
{
write_path_index = 0;
}


// 判断storage的store_path剩余空间，获取充足空间的store_path。
avg_reserved_mb = g_storage_reserved_space.rs.mb / \
  pStoreGroup->store_path_count;
if (!tracker_check_reserved_space_path(pStorageServer-> \
path_total_mbs[write_path_index], pStorageServer-> \
path_free_mbs[write_path_index], avg_reserved_mb))
{
int i;
for (i=0; i<pStoreGroup->store_path_count; i++)
{
if (tracker_check_reserved_space_path( \
pStorageServer->path_total_mbs[i], \
pStorageServer->path_free_mbs[i], \
avg_reserved_mb))
{
pStorageServer->current_write_path = i;
write_path_index = i;
break;
}
}

if (i == pStoreGroup->store_path_count)
{
if (!g_if_use_trunk_file)
{
pTask->length = sizeof(TrackerHeader);
return ENOSPC;
}

for (i=write_path_index; i<pStoreGroup-> \
store_path_count; i++)
{
if (tracker_check_reserved_space_path( \
  pStorageServer->path_total_mbs[i], \
  pStorageServer->path_free_mbs[i] + \
  pStoreGroup->trunk_free_mb, \
  avg_reserved_mb))
{
pStorageServer->current_write_path = i;
write_path_index = i;
break;
}
}
if ( i == pStoreGroup->store_path_count)
{
for (i=0; i<write_path_index; i++)
{
      if (tracker_check_reserved_space_path( \
              pStorageServer->path_total_mbs[i], \
              pStorageServer->path_free_mbs[i] + \
              pStoreGroup->trunk_free_mb, \
              avg_reserved_mb))
    {
pStorageServer->current_write_path = i;
write_path_index = i;
break;
    }
}

if (i == write_path_index)
{
pTask->length = sizeof(TrackerHeader);
return ENOSPC;
}
}
}
}

// 获取到store_path后，更新下次获取的位置
if (g_groups.store_path == FDFS_STORE_PATH_ROUND_ROBIN)
{
pStorageServer->current_write_path++;
if (pStorageServer->current_write_path >= \
pStoreGroup->store_path_count)
{
pStorageServer->current_write_path = 0;
}
}


/*
//printf("pStoreGroup->current_write_server: %d, " \
"pStoreGroup->active_count=%d\n", \
pStoreGroup->current_write_server, \
pStoreGroup->active_count);
*/
/* 获取到storage和store_path后，填写返回信息
包括：组名，storage信息，store_path标志。
*/

        // 5. 组装返回包信息

p = pTask->data + sizeof(TrackerHeader);

        // 存储组名

memcpy(p, pStoreGroup->group_name, FDFS_GROUP_NAME_MAX_LEN);
p += FDFS_GROUP_NAME_MAX_LEN;

// 如果是获取整组的storage，则填写整组active的storage信息，否则仅填写获取到的storage。
if (cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ALL
|| cmd == TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ALL)
{
int active_count;
FDFSStorageDetail **ppServer;
FDFSStorageDetail **ppEnd;

active_count = pStoreGroup->active_count;
if (active_count == 0)
{
pTask->length = sizeof(TrackerHeader);
return ENOENT;
}

ppEnd = pStoreGroup->active_servers + active_count;
for (ppServer=pStoreGroup->active_servers; ppServer<ppEnd; \
ppServer++)
{
memcpy(p, (*ppServer)->ip_addr, IP_ADDRESS_SIZE - 1);
p += IP_ADDRESS_SIZE - 1;

long2buff(pStoreGroup->storage_port, p);
p += FDFS_PROTO_PKG_LEN_SIZE;
}
}
else
{
memcpy(p, pStorageServer->ip_addr, IP_ADDRESS_SIZE - 1);
p += IP_ADDRESS_SIZE - 1;

long2buff(pStoreGroup->storage_port, p);
p += FDFS_PROTO_PKG_LEN_SIZE;
}

*p++ = (char)write_path_index;
pTask->length = p - pTask->data;

return 0;
}

根据源代码解析，最终选择到的storage是active storage，其存储空间充足。选中的store_path也是空间充足的路径。

4. storage upload相关源码解析
upload文件在与tracker通信获取storage后，先与storage连接，再上传文件。其中与storage连接为socket连接；上传文件（文件上传方式），先发送包头，发送了命令：STORAGE_PROTO_CMD_UPLOAD_FILE，再使用sendfile函数上传文件。
在storage端，由accept线程接收连接，通知nio网络线程接收数据。nio线程解析处理后，由dio磁盘线程写入文件。
accept线接收连接，nio线程接收数据等属于storage框架内容，这里不深入分析。我们从nio线程解析处理代码开始分析。
nio线程解析处理函数为storage_deal_task[storage/storage_service.c]，其中与upload相关的为：
case STORAGE_PROTO_CMD_UPLOAD_FILE:
ACCESS_LOG_INIT_FIELDS();
result = storage_upload_file(pTask, false);
STORAGE_ACCESS_LOG(pTask, \
ACCESS_LOG_ACTION_UPLOAD_FILE, \
result);
break;
其调用了storage_upload_file[storage/storage_service.c]函数。
4.1 storage_upload_file函数
storage/storage_service.c
/**
1 byte: store path index
8 bytes: file size 
FDFS_FILE_EXT_NAME_MAX_LEN bytes: file ext name, do not include dot (.)
file size bytes: file content
**/
static int storage_upload_file(struct fast_task_info *pTask, bool bAppenderFile)
{
StorageClientInfo *pClientInfo;
StorageFileContext *pFileContext;
DisconnectCleanFunc clean_func;
char *p;
char filename[128];
char file_ext_name[FDFS_FILE_PREFIX_MAX_LEN + 1];
int64_t nInPackLen;
int64_t file_offset;
int64_t file_bytes;
int crc32;
int store_path_index;
int result;
int filename_len;

// 1. 获取连接信息，判断包长是否正确。
pClientInfo = (StorageClientInfo *)pTask->arg;
pFileContext =  &(pClientInfo->file_context);
nInPackLen = pClientInfo->total_length - sizeof(TrackerHeader);

if (nInPackLen < 1 + FDFS_PROTO_PKG_LEN_SIZE + 
FDFS_FILE_EXT_NAME_MAX_LEN)
{
logError("file: "__FILE__", line: %d, " \
"cmd=%d, client ip: %s, package size " \
"%"PRId64" is not correct, " \
"expect length >= %d", __LINE__, \
STORAGE_PROTO_CMD_UPLOAD_FILE, \
pTask->client_ip,  nInPackLen, \
1 + FDFS_PROTO_PKG_LEN_SIZE + \
FDFS_FILE_EXT_NAME_MAX_LEN);
return EINVAL;
}

// 2. 获取store_path。
/* 如果为-1，重新获取store_path_index；如果小于0或者大于store_path个数，返回错误*/
p = pTask->data + sizeof(TrackerHeader);
store_path_index = *p++;

if (store_path_index == -1)
{
// 获取存储充足的store_path
if ((result=storage_get_storage_path_index( \
&store_path_index)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"get_storage_path_index fail, " \
"errno: %d, error info: %s", __LINE__, \
result, STRERROR(result));
return result;
}
}
else if (store_path_index < 0 || store_path_index >= \
g_fdfs_store_paths.count)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, store_path_index: %d " \
"is invalid", __LINE__, \
pTask->client_ip, store_path_index);
return EINVAL;
}

// 3.获取文件大小
file_bytes = buff2long(p);
p += FDFS_PROTO_PKG_LEN_SIZE;
if (file_bytes < 0 || file_bytes != nInPackLen - \
(1 + FDFS_PROTO_PKG_LEN_SIZE + \
FDFS_FILE_EXT_NAME_MAX_LEN))
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, pkg length is not correct, " \
"invalid file bytes: %"PRId64 \
", total body length: %"PRId64, \
__LINE__, pTask->client_ip, file_bytes, nInPackLen);
return EINVAL;
}

// 4.获取文件后缀
memcpy(file_ext_name, p, FDFS_FILE_EXT_NAME_MAX_LEN);
*(file_ext_name + FDFS_FILE_EXT_NAME_MAX_LEN) = '\0';
p += FDFS_FILE_EXT_NAME_MAX_LEN;
if ((result=fdfs_validate_filename(file_ext_name)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, file_ext_name: %s " \
"is invalid!", __LINE__, \
pTask->client_ip, file_ext_name);
return result;
}

// 5.封装pFileContext
pFileContext->calc_crc32 = true;
pFileContext->calc_file_hash = g_check_file_duplicate;
pFileContext->extra_info.upload.start_time = g_current_time;

strcpy(pFileContext->extra_info.upload.file_ext_name, file_ext_name);
storage_format_ext_name(file_ext_name, \
pFileContext->extra_info.upload.formatted_ext_name);
pFileContext->extra_info.upload.trunk_info.path. \
store_path_index = store_path_index;
pFileContext->extra_info.upload.file_type = _FILE_TYPE_REGULAR;
pFileContext->sync_flag = STORAGE_OP_TYPE_SOURCE_CREATE_FILE;
pFileContext->timestamp2log = pFileContext->extra_info.upload.start_time;
pFileContext->op = FDFS_STORAGE_FILE_OP_WRITE;
if (bAppenderFile)
{
// 分块上传
pFileContext->extra_info.upload.file_type |= \
_FILE_TYPE_APPENDER;
}
else
{
// 判断是否使用“合并存储”
if (g_if_use_trunk_file && trunk_check_size( \
TRUNK_CALC_SIZE(file_bytes)))
{
pFileContext->extra_info.upload.file_type |= \
_FILE_TYPE_TRUNK;
}
}

// 如果使用“合并存储”
if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_TRUNK)
{
FDFSTrunkFullInfo *pTrunkInfo;

pFileContext->extra_info.upload.if_sub_path_alloced = true;
pTrunkInfo = &(pFileContext->extra_info.upload.trunk_info);
if ((result=trunk_client_trunk_alloc_space( \
TRUNK_CALC_SIZE(file_bytes), pTrunkInfo)) != 0)
{
return result;
}

clean_func = dio_trunk_write_finish_clean_up;
file_offset = TRUNK_FILE_START_OFFSET((*pTrunkInfo));
pFileContext->extra_info.upload.if_gen_filename = true;
trunk_get_full_filename(pTrunkInfo, pFileContext->filename, \
sizeof(pFileContext->filename));
pFileContext->extra_info.upload.before_open_callback = \
dio_check_trunk_file_when_upload;
pFileContext->extra_info.upload.before_close_callback = \
dio_write_chunk_header;
pFileContext->open_flags = O_RDWR | g_extra_open_file_flags;
}
else // 非“合并存储”情况
{
char reserved_space_str[32];

// 判断剩余空间是否满足当前文件存储
if (!storage_check_reserved_space_path(g_path_space_list \
[store_path_index].total_mb, g_path_space_list \
[store_path_index].free_mb - (file_bytes/FDFS_ONE_MB), \
g_avg_storage_reserved_mb))
{
logError("file: "__FILE__", line: %d, " \
"no space to upload file, "
"free space: %d MB is too small, file bytes: " \
"%"PRId64", reserved space: %s", \
__LINE__, g_path_space_list[store_path_index].\
free_mb, file_bytes, \
fdfs_storage_reserved_space_to_string_ex( \
g_storage_reserved_space.flag, \
g_avg_storage_reserved_mb, \
g_path_space_list[store_path_index]. \
total_mb, g_storage_reserved_space.rs.ratio,\
reserved_space_str));
return ENOSPC;
}


// 生成文件名称
crc32 = rand();
*filename = '\0';
filename_len = 0;
pFileContext->extra_info.upload.if_sub_path_alloced = false;
if ((result=storage_get_filename(pClientInfo, \
pFileContext->extra_info.upload.start_time, \
file_bytes, crc32, pFileContext->extra_info.upload.\
formatted_ext_name, filename, &filename_len, \
pFileContext->filename)) != 0)
{
return result;
}

// dio线程处理函数中用到的清除函数clean_func
// 关闭打开的pFileContext fd，判断如果未完成写入，就删除该文件。
clean_func = dio_write_finish_clean_up;
file_offset = 0;
pFileContext->extra_info.upload.if_gen_filename = true;
pFileContext->extra_info.upload.before_open_callback = NULL;
pFileContext->extra_info.upload.before_close_callback = NULL;
pFileContext->open_flags = O_WRONLY | O_CREAT | O_TRUNC \
| g_extra_open_file_flags;
}

/* 6.继续封装pFileContext，并把pTask压入dio队列。
包括封装文件内容，大小，偏移量，任务处理函数, 完成回调函数，清除函数
封装后，把这个任务压入dio线程的队列，再由dio线程读取队列，调用任务处理函数处理任务
任务处理函数:dio_write_file 把上传内容写入文件，更新文件相关的属性。如果失败，调用清除函数。最后调用完成回调函数。
完成回调函数：storage_upload_file_done_callback 封装pFileContext之中的文件上传的组名，fileid，记录binlog等。
清除函数：dio_write_finish_clean_up 上面有介绍
*/
return storage_write_to_file(pTask, file_offset, file_bytes, \
p - pTask->data, dio_write_file, \
storage_upload_file_done_callback, \
clean_func, store_path_index);
}


其中主要的回调函数为：dio_write_file[storage/storage_dio.c]，storage_upload_file_done_callback[storage_service.c]，dio_write_finish_clean_up[storage/storage_dio.c]。
4.2 dio_write_file函数
在storage_write_to_file函数中，封装pFileContext，也就封装了pTask。并且把这个pTask压入dio线程的队列中。在dio线程dio_thread_entrance[storage/storage_dio.c]函数中，读取队列任务，调用了回调函数，如下：
pContext = (struct storage_dio_context *)arg; 
while (g_continue_flag)
{
while ((pTask=blocked_queue_pop(&(pContext->queue))) != NULL)
{
((StorageClientInfo *)pTask->arg)->deal_func(pTask);
}
}
此时，在这里的deal_func就是dio_write_file函数。

storage/storage_dio.c
int dio_write_file(struct fast_task_info *pTask)
{
StorageClientInfo *pClientInfo;
StorageFileContext *pFileContext;
int result;
int write_bytes;
char *pDataBuff;

// 1.获取client信息和文件信息
pClientInfo = (StorageClientInfo *)pTask->arg;
pFileContext = &(pClientInfo->file_context);
result = 0;
do
{
// 2.打开文件写入的fd
if (pFileContext->fd < 0)
{
// 如果注册打开前回调函数，即调用处理。在upload过程中，无注册。
if (pFileContext->extra_info.upload.before_open_callback!=NULL)
{
result = pFileContext->extra_info.upload. \
before_open_callback(pTask);
if (result != 0)
{
break;
}
}
// 根据文件路径打开文件，并设置偏移量。
if ((result=dio_open_file(pFileContext)) != 0)
{
break;
}
}

// 3.写入文件内容
pDataBuff = pTask->data + pFileContext->buff_offset;
write_bytes = pTask->length - pFileContext->buff_offset;
if (fc_safe_write(pFileContext->fd, pDataBuff, write_bytes) != write_bytes)
{
result = errno != 0 ? errno : EIO;
logError("file: "__FILE__", line: %d, " \
"write to file: %s fail, fd=%d, write_bytes=%d, " \
"errno: %d, error info: %s", \
__LINE__, pFileContext->filename, \
pFileContext->fd, write_bytes, \
result, STRERROR(result));
}

// 4.更新storage status
pthread_mutex_lock(&g_dio_thread_lock);
g_storage_stat.total_file_write_count++;
if (result == 0)
{
g_storage_stat.success_file_write_count++;
}
pthread_mutex_unlock(&g_dio_thread_lock);


if (result != 0)
{
break;
}


// 5.文件校验信息
if (pFileContext->calc_crc32)
{
pFileContext->crc32 = CRC32_ex(pDataBuff, write_bytes, \
pFileContext->crc32);
}

if (pFileContext->calc_file_hash)
{
if (g_file_signature_method == STORAGE_FILE_SIGNATURE_METHOD_HASH)
{
CALC_HASH_CODES4(pDataBuff, write_bytes, \
pFileContext->file_hash_codes)
}
else
{
my_md5_update(&pFileContext->md5_context, \
(unsigned char *)pDataBuff, write_bytes);
}
}

/*
logInfo("###dio write bytes: %d, pTask->length=%d, buff_offset=%d", \
write_bytes, pTask->length, pFileContext->buff_offset);
*/

pFileContext->offset += write_bytes;
if (pFileContext->offset < pFileContext->end)
{
// 文件写入不完整，通知nio线程处理【继续读取信息】
pFileContext->buff_offset = 0;
storage_nio_notify(pTask);  //notify nio to deal
}
else
{
// 文件写入成功，封装校验信息
if (pFileContext->calc_crc32)
{
pFileContext->crc32 = CRC32_FINAL( \
pFileContext->crc32);
}

if (pFileContext->calc_file_hash)
{
if (g_file_signature_method == STORAGE_FILE_SIGNATURE_METHOD_HASH)
{
FINISH_HASH_CODES4(pFileContext->file_hash_codes)
}
else
{
my_md5_final((unsigned char *)(pFileContext-> \
file_hash_codes), &pFileContext->md5_context);
}
}

// 如果注册了关闭前回调函数，即调用处理。在upload过程中，无注册。
if (pFileContext->extra_info.upload.before_close_callback != NULL)
{
result = pFileContext->extra_info.upload. \
before_close_callback(pTask);
}

// 关闭打开的fd
/* file write done, close it */
close(pFileContext->fd);
pFileContext->fd = -1;

// 如果注册完成回调函数，即调用处理。在upload过程中，注册storage_upload_file_done_callback。
if (pFileContext->done_callback != NULL)
{
pFileContext->done_callback(pTask, result);
}
}

return 0;
} while (0);


// 失败返回。调用清除函数。在upload过程中注册了dio_write_finish_clean_up。
pClientInfo->clean_func(pTask);
// 如果注册完成回调函数，即调用处理。
if (pFileContext->done_callback != NULL)
{
pFileContext->done_callback(pTask, result);
}
return result;
}
4.3 storage_upload_file_done_callback函数
在upload的过程中，dio_write_file中任务完成后调用done_callback ，在这里done_callback就是storage_upload_file_done_callback函数。
storage/storage_service.c
static void storage_upload_file_done_callback(struct fast_task_info *pTask, \
const int err_no)
{
StorageClientInfo *pClientInfo;
StorageFileContext *pFileContext;
TrackerHeader *pHeader;
int result;

// 1.获取client信息和文件信息
pClientInfo = (StorageClientInfo *)pTask->arg;
pFileContext =  &(pClientInfo->file_context);

// 2.确定返回码
if (pFileContext->extra_info.upload.file_type & _FILE_TYPE_TRUNK)
{
result = trunk_client_trunk_alloc_confirm( \
&(pFileContext->extra_info.upload.trunk_info), err_no);
if (err_no != 0)
{
result = err_no;
}
}
else
{
result = err_no;
}

// 3.写binlog，确定需要返回的组名和文件id等
if (result == 0)
{
result = storage_service_upload_file_done(pTask);
if (result == 0)
{
if (pFileContext->create_flag & STORAGE_CREATE_FLAG_FILE)
{
result = storage_binlog_write(\
pFileContext->timestamp2log, \
STORAGE_OP_TYPE_SOURCE_CREATE_FILE, \
pFileContext->fname2log);
}
}
}

// 4.更新storage status。
if (result == 0)
{
int filename_len;
char *p;

if (pFileContext->create_flag & STORAGE_CREATE_FLAG_FILE)
{
CHECK_AND_WRITE_TO_STAT_FILE3_WITH_BYTES( \
g_storage_stat.total_upload_count, \
g_storage_stat.success_upload_count, \
g_storage_stat.last_source_update, \
g_storage_stat.total_upload_bytes, \
g_storage_stat.success_upload_bytes, \
pFileContext->end - pFileContext->start)
}

filename_len = strlen(pFileContext->fname2log);
pClientInfo->total_length = sizeof(TrackerHeader) + \
FDFS_GROUP_NAME_MAX_LEN + filename_len;
p = pTask->data + sizeof(TrackerHeader);
memcpy(p, pFileContext->extra_info.upload.group_name, \
FDFS_GROUP_NAME_MAX_LEN);
p += FDFS_GROUP_NAME_MAX_LEN;
memcpy(p, pFileContext->fname2log, filename_len);
}
else
{
pthread_mutex_lock(&stat_count_thread_lock);
if (pFileContext->create_flag & STORAGE_CREATE_FLAG_FILE)
{
g_storage_stat.total_upload_count++;
  g_storage_stat.total_upload_bytes += \
pClientInfo->total_offset;
}
pthread_mutex_unlock(&stat_count_thread_lock);

pClientInfo->total_length = sizeof(TrackerHeader);
}

STORAGE_ACCESS_LOG(pTask, ACCESS_LOG_ACTION_UPLOAD_FILE, result);

// 5.封装dio返回给nio的pTask。
pClientInfo->total_offset = 0;
pTask->length = pClientInfo->total_length;

pHeader = (TrackerHeader *)pTask->data;
pHeader->status = result;
pHeader->cmd = STORAGE_PROTO_CMD_RESP;
long2buff(pClientInfo->total_length - sizeof(TrackerHeader), \
pHeader->pkg_len);

        // 6.通知nio线程处理【此时，包已接收完毕，直接网络发送给client。】
storage_nio_notify(pTask);
}