server.cpp
#include "httplib.h"
#include <boost/filesystem.hpp>
#include <iostream>
#include <sstream>
#include <fstream>
#if 0
void HelloWorld(const httplib::Request &req, httplib::Response &rsp)
{
//text/html 顯示在瀏覽器上的
rsp.set_content("<html><h1>Hello World</h1></html>", "text/html");
}
int main()
{
httplib::Server srv;
//設置相對根目錄
srv.set_base_dir("./www");
//當客戶端發送get請求,該如何去處理
// 192.168.43.85:9000/
srv.Get("/", HelloWorld);
srv.listen("0.0.0.0", 9000);
return 0;
}
#endif
#define SERVER_BASE_DIR "www"
#define SERVER_ADDR "0.0.0.0"
#define SERVER_PORT 9000
#define SERVER_BACKUP_DIR SERVER_BASE_DIR"/list/"
using namespace httplib;
namespace bf = boost::filesystem;
class CloudServer{
private:
httplib::Server srv;
public:
CloudServer(){
bf::path base_path(SERVER_BASE_DIR);
//如果文件不存在
if (!bf::exists(base_path)){
//創建目錄
bf::create_directory(base_path);
}
bf::path list_path(SERVER_BACKUP_DIR);
if (!bf::exists(list_path)){
bf::create_directory(list_path);
}
}
bool Start(){
srv.set_base_dir(SERVER_BASE_DIR);
srv.Get("/(list(/){0,1}){0,1}", GetFileList);
srv.Get("/list/(.*)", GetFileData);
srv.Put("/list/(.*)", PutFileData);
srv.listen("SERVER_ADDR", SERVER_PORT);
return true;
}
private:
static void PutFileData(const Request &req, httplib::Response &rsp)
{
std::cout << "backup file " << req.path << "\n";
if (!req.has_header("Range")){
rsp.status = 400;
return;
}
std::string range = req.get_header_value("Range");
int64_t range_start;
if (RangeParse(range, range_start) == false){
rsp.status = 400;
return;
}
std::string real = SERVER_BASE_DIR + req.path;
std::ofstream file(real, std::ios::binary | std::ios::trunc);
if (!file.is_open()){
std::cerr << "open file " << real << "error\n";
rsp.status = 500;
return;
}
file.seekp(range_start, std::ios::beg);
file.write(&req.body[0], req.body.size());
if (!file.good()){
std::cerr << "file write body error\n";
rsp.status = 500;
return;
}
file.close();
return;
}
static bool RangeParse(std::string &range, int64_t &start){
//Range: bytes=start-end
size_t pos1 = range.find("=");
size_t pos2 = range.find("=");
if (pos1 == std::string::npos || pos2 == std::string::npos){
std::cerr << "range:[" << range << "] format error\n";
return false;
}
std::stringstream rs;
rs << range.substr(pos1 + 1, pos2 - pos1 - 1);
rs >> start;
return true;
}
//文件列表信息獲取,定義成靜態就是避免了this指針
static void GetFileList(const Request &req, httplib::Response &rsp){
bf::path list(SERVER_BACKUP_DIR);
bf::directory_iterator item_begin(list);
bf::directory_iterator item_end;
std::string body;
body = "<html><body><ol><hr />";
for (; item_begin != item_end; ++item_begin){
//如果不是目錄
if (bf::is_directory(item_begin->status())){
continue;
}
std::string file = item_begin->path().filename().string();
std::string uri = "/list/" + file;
body += "<h4><li>";
body += "<a href='";
body += uri;
body += "'>";
body += file;
body += "</a>";
body += "</li></h4>";
//std::string file = item_begin->path().string();
//std::cerr << "file:" << file << std::endl;
}
body += "<hr /></ol></body></html>";
rsp.set_content(&body[0], "text/html");
return;
}
//獲取文件數據(文件下載)
static void GetFileData(const Request &req, httplib::Response &rsp){
//req.path = "/list/a.txt";
std::string file = SERVER_BACKUP_DIR + req.path;
//文件不存在,page not found
if (!bf::exists(file)){
std::cerr << "file " << file << "is not exists\n";
rsp.status = 404;
return;
}
//文件存在
//ifile = inputfile
std::ifstream ifile(file, std::ios::binary);
if (!ifile.is_open()){
std::cerr << "open file " << file << "error\n";
rsp.status = 500; //服務器內部錯誤
return;
}
std::string body;
//獲取文件大小
int64_t fsize = bf::file_size(file);
body.resize(fsize);
ifile.read(&body[0], fsize);
if (ifile.good()){
std::cerr << "read file " << file << "body error\n";
rsp.status = 500;
return;
}
//正文只能給一次,文件不能太大
rsp.set_content(body, "text/plain"); //plain文件下載
}
//解決文件上傳
static void BackuoFile(const Request &req, httplib::Response &rsp){
}
};
int main()
{
CloudServer srv;
srv.Start();
return 0;
}
CloudClient.hpp
#define _D_SCL_SECURE_NO_WARNINGS 1
#ifndef __M_CLOUD_H__
#define __M_CLOUD_H__
#include <fstream>
#include <sstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>
#include <thread>
#include "httplib.h"
#define SERVER_IP "192.168.122.132"
#define SERVER_PORT 9000
#define CLIENT_BACKUP_DIR ".\\backup"
#define CLIENT_BACKUP_INFO_FILE ".\\back.list"
#define RANGE_MAX_SIZE (10 << 20) //10MB
#define BACKUP_URI "/list/"
namespace bf = boost::filesystem;
class ThrBackUp
{
private:
std::string _file;
int64_t _range_start;
int64_t _range_len;
public:
bool _res;
public:
ThrBackUp(const std::string &file, int64_t start, int64_t len) :_res(true),
_file(file), _range_start(start), _range_len(len) {}
void Start(){
//獲取文件的range分塊數據
std::stringstream ss;
ss << "backup file" << _file << "range:[" << _range_start << "-" << _range_len << "\n";
std::cout << ss.str();
std::ifstream path(_file, std::ios::binary);
if (!path.is_open()){
std::cerr << "range backup file " << _file << "failed!\n";
_res = false;
return;
}
//跳轉到range的起始位置
path.seekg(_range_start, std::ios::beg);//跳轉
std::string body;
body.resize(_range_len);
//讀取文件中range分塊的文件數據
path.read(&body[0], _range_len);
if (!path.good()){
std::cerr << "read file" << _file << " range data failed\n";
_res = false;
return;
}
path.close();
//上傳range數據
bf::path name(_file);
//組織上傳的uri路徑 method url version
//PUT /list/filename HTTP/1.1
std::string url = BACKUP_URI + name.filename().string();//native 不要雙引號
//實例化一個httplib的客戶端對象
httplib::Client cli(SERVER_IP, SERVER_PORT);
//定義HTTP請求頭信息
httplib::Headers hdr;
hdr.insert(std::make_pair("Content-Length", std::to_string(_range_len)));
std::stringstream tmp;
tmp << "bytes=" << _range_start << "-" << _range_start + _range_len - 1;
hdr.insert(std::make_pair("Range", tmp.str().c_str()));
//通過實例化的Client向服務端發送PUT請求
auto rsp = cli.Put(url.c_str(), hdr, body, "text/plain");
if (rsp &&rsp->status != 200){
_res = false;
}
else {
std::stringstream ss;
ss << "backup file" << _file << "] range:[" << _range_start << "-" << _range_len << "] backup success\n";
std::cout << ss.str();
return;
}
//return ;
}
};
class CloudClient
{
public:
CloudClient(){
bf::path file(CLIENT_BACKUP_DIR);
if (!bf::exists(file)){
bf::create_directory(file);
}
}
private:
std::unordered_map<std::string, std::string> _backup_list;
//CloudClient(){}
private:
//獲取備份信息
bool GetBackupInfo()
{
//filename1 etag\n
//filename2 etag\n
//實例化path對象
bf::path path(CLIENT_BACKUP_INFO_FILE);
if (!bf::exists(path)){
std::cerr << "list file" << path.string() << "is not exists\n";
return false;
}
int64_t fsize = bf::file_size(path);
if (fsize == 0){
std::cerr << "have no backup info\n";
return false;
}
std::string body;
body.resize(fsize);
std::ifstream file(CLIENT_BACKUP_INFO_FILE, std::ios::binary);
if (!file.is_open()){
std::cerr << "list file open error\n";
return false;
}
file.read(&body[0], fsize);
if (!file.good()){
std::cerr << "read list file body error\n";
return false;
}
file.close();
std::vector<std::string> list;
boost::split(list, body, boost::is_any_of("\n"));//切割後放到list中
for (auto i : list){
//filename2 etag
size_t pos = i.find(" ");
if (pos == std::string::npos)
continue;
std::string key = i.substr(0, pos);
std::string val = i.substr(pos + 1);
_backup_list[key] = val;
}
return true;
}
//將備份信息保存在指定文件中
bool SetBackuoInfo()
{
std::string body;
for (auto i : _backup_list){
body += i.first + " " + i.second + "\n";
}
std::ofstream file(CLIENT_BACKUP_INFO_FILE, std::ios::binary);
if (!file.is_open()){
std::cerr << "open list file error\n";
return false;
}
file.write(&body[0], body.size());
if (!file.good()){
std::cerr << "set backuo info error\n";
return false;
}
file.close();
return true;
}
//目錄監控
bool BackupDirListen(const std::string &path)
{
bf::directory_iterator item_begin(path);
bf::directory_iterator item_end;
for (; item_begin != item_end; ++item_begin){
// /list/abc/abc.txt bc.txt
if (bf::is_directory(item_begin->status())){
BackupDirListen(item_begin->path().string());
continue;
}
if (FileIsNeedBackup(item_begin->path().string()) == false){
continue;
}
std::cerr << "file:[" << item_begin->path().string() << "need backup\n";
if (PutFileData(item_begin->path().string()) == false){
continue;
}
//上傳成功後
//記錄文件Etag信息
AddBackInfo(item_begin->path().string());
}
return true;
}
//添加一個信息
bool AddBackInfo(const std::string &file)
{
//etag = "mtime-fsize"
std::string etag;
if (GetFileEtag(file, etag) == false)
return false;
_backup_list[file] = etag;
}
static void thr_start(ThrBackUp *backuo_info){
/*
*/
return;
}
//備份文件
bool PutFileData(const std::string &file)
{
//按大小對文件內容進行分塊傳輸(10MB)
//通過獲取分塊傳輸是否成功判斷整個文件是否上傳成功
//選擇多線程處理
//1.獲取文件大小
int64_t fsize = bf::file_size(file);
if (fsize <= 0){
std::cerr << "file " << file << "unnecessary backup\n";
return false;
}
//2.計算總共需要分多少塊,得到每塊大小以及起始位置
//3.循環創建線程,在線程中上傳文件數據
int count = (int)fsize / RANGE_MAX_SIZE;
std::vector<ThrBackUp>thr_res;
std::vector<std::thread> thr_list;
std::cerr << "file:[" << file << "]fsize:[" << fsize << "] count:[" << count + 1 << "]\n";
// fsize = 500;
for (int i = 0; i <= count; ++i){
int64_t range_start = i * RANGE_MAX_SIZE; //起始位置
int64_t range_end = (i + 1) * RANGE_MAX_SIZE - 1;
if (i == count-1){
range_end += (fsize % RANGE_MAX_SIZE);
}
int64_t range_len = range_end - range_start + 1;
ThrBackUp backup_info(file, range_start, range_len);
std::cerr << "file:[" << file << "] range:[" << range_start << "-" << range_end << "]-" << range_len << "\n";
thr_res.push_back(backup_info);
thr_list.push_back(std::thread(thr_start, &thr_res[i]));
//std::thread thr(thr_start, file, range_start, range_len, &thr_res[i]);
}
/*for (int i = 0; i <= count; ++i)
{
thr_list.push_back(std::thread(thr_start, &thr_res[i]));
}*/
//4.等待所有線程退出,判斷文件上傳結果
bool ret = true;
for (int i = 0; i <= count; ++i){
thr_list[i].join();
if (thr_res[i]._res == true)
continue;
}
ret = false;
//5.上傳成功,則添加文件的備份信息記錄。 native() //不要雙引號
if (ret == false){
return false;
}
//AddBackInfo(file);
std::cerr << "file:[" << file << "] backup success\n";
return true;
}
bool FileIsNeedBackup(const std::string &file)
{
std::string etag;
if (GetFileEtag(file, etag) == false)
return false;
auto it = _backup_list.find(file);
if (it != _backup_list.end() && it->second == etag){
//不需要備份
return false;
}
return true;
}
bool GetFileEtag(const std::string &file, std::string &etag)
{
bf::path path(file);
if (!bf::exists(path))
{
std::cerr << "get file" << file << "etag error\n";
return false;
}
int64_t fsize = bf::file_size(path);
int64_t mtime = bf::last_write_time(path);
std::stringstream tmp;
tmp << std::hex << fsize << "-" << std::hex << mtime;
etag = tmp.str();
return true;
}
static void thr_start(ThrBackUp *backup_info)
{
backup_info->Start();
return;
}
public:
bool Start()
{
GetBackupInfo();//獲取備份信息
while (1)
{
//對文件目錄進行監控
BackupDirListen(CLIENT_BACKUP_DIR);
SetBackuoInfo();
Sleep(10);
}
return true;
}
};
#endif
compress.hpp
#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
#include <unordered_map>
#include <vector>
#include <thread>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <zlib.h>
#include <pthread.h>
#include <sys/file.h>
//未壓縮文件存儲位置
#define UNGZIPFILE_PATH "www/list/"
//壓縮包存儲位置
#define GZIPFILE_PATH "www/zip/"
//存放文件信息(filename gzipfilename\n)
#define RECORD_FILE "record.list"
//熱度過期時間
#define HEAT_TIME 2
namespace bf = boost::filesystem;
class CompressStore
{
private:
//用於保存文件列表
std::unordered_map<std::string, std::string> _file_list;
pthread_rwlock_t _rwlock;
private:
//1. 每次壓縮存儲線程啓動的時候,從文件中讀取列表信息
bool GetListRecord() {
//存儲列表信息格式如下:
//filename gzipfilename\n
bf::path name(RECORD_FILE);
if(!bf::exists(name)) {
std::cerr << "record file is not exists\n";
return false;
}
std::ifstream file(RECORD_FILE, std::ios::binary);
if(!file.is_open()){
std::cerr << "open record file error\n";
return false;
}
int64_t fsize = bf::file_size(name);
std::string body;
body.resize(fsize);
file.read(&body[0], fsize);
if(!file.good()){
std::cerr << "record file body read error\n";
return false;
}
file.close();
std::vector<std::string> list;
boost::split(list, body, boost::is_any_of("\n"));
for(auto e : list) {
//分割後的信息如下
//filename gzipfilename
size_t pos = e.find(" ");
if(pos == std::string::npos) {
continue;
}
std::string key = e.substr(0, pos);
std::string val = e.substr(pos + 1);
_file_list[key] = val;
}
return true;
}
//2. 每次壓縮存儲完畢,都要將列表信息,存儲到文件中
bool SetListRecord() {
std::stringstream tmp;
for(auto i : _file_list) {
tmp << i.first << " " << i.second << "\n";
}
std::ofstream file(RECORD_FILE, std::ios::binary | std::ios::trunc);
if(!file.is_open()) {
std::cerr << "record file open error\n";
return false;
}
file.write(tmp.str().c_str(), tmp.str().size());
if(!file.good()) {
std::cerr << "record file write body error\n";
return false;
}
file.close();
return true;
}
//目錄檢測,獲取目錄中的文件名
// 1.判斷文件是否需要壓縮存儲
// 2.文件壓縮存儲
bool DirectoryCheck() {
if(!bf::exists(UNGZIPFILE_PATH)) {
bf::create_directory(UNGZIPFILE_PATH);
}
std::string cmd = "bash gbk_utf8";
FILE *fd = popen(cmd.c_str(),"r");
pclose(fd);
bf::directory_iterator item_begin(UNGZIPFILE_PATH);
bf::directory_iterator item_end;
for(; item_begin != item_end; ++item_begin) {
if(bf::is_directory(item_begin->status())) {
continue;
}
std::string name = item_begin->path().string();
if(!IsNeedCompress(name)) {
continue;
}
std::string gzip = GZIPFILE_PATH + item_begin->path().filename().string() + ".gz";
if(CompressFile(name, gzip)) {
std::cerr << "file:[" << name << "] store success\n";
AddFileRecord(name, gzip);
}else {
std::cerr << "file:[" << name << "] store failed\n";
}
}
return true;
}
//2.2. 判斷文件是否需要壓縮存儲
bool IsNeedCompress(std::string &file) {
//通過stat這個系統調用接口可以獲取文件相關信息,取最後一次訪問時間
struct stat st;
if(stat(file.c_str(), &st) < 0) {
std::cerr << "get file:[" << file << "] stat error\n";
return false;
}
time_t cur_time = time(nullptr); //當前系統時間
time_t acc_time = st.st_atime;
if((cur_time - acc_time) < HEAT_TIME){
return false;
}
return true;
}
//2.3. 對文件進行壓縮存儲
bool CompressFile(std::string &file, std::string &gzip) {
int fd = open(file.c_str(), O_RDONLY);
if(fd < 0) {
std::cerr << "com open file:[" << file << "] error\n";
return false;
}
gzFile gf = gzopen(gzip.c_str(), "wb");
if(gf == nullptr) {
std::cerr << "com open gzip:[" << gzip<< "] error\n";
return false;
}
int ret;
char buf[1024];
flock(fd, LOCK_SH);
while((ret = read(fd, buf, 1024)) > 0) {
gzwrite(gf, buf, ret);
}
flock(fd, LOCK_UN);
close(fd);
gzclose(gf);
//壓縮完後,原文件就沒必要存在了,所以調用unlink刪除原文件
//int unlink(const char *pathname);
//如果文件正在被操作,那麼會報錯:EBUSY
//EBUSY:The file pathname cannot be unlinked because it is being used by the system or another process; for example, it is a mount point or the NFS client software created it to represent an active but otherwise nameless inode ("NFS silly renamed").
unlink(file.c_str()); //unlink 在刪除文件時,如果這個文件已經被打開且正在被操作,這個文件就無法刪除
return true;
}
//對文件解壓縮
bool UnCompressFile(std::string &gzip, std::string &file) {
int fd = open(file.c_str(), O_CREAT | O_WRONLY, 0664);
if(fd < 0) {
std::cerr << "open file" << file << "failed\n";
return false;
}
gzFile gf = gzopen(gzip.c_str(), "rb");
if(gf == nullptr) {
std::cerr << "open gzip " << gzip << "failed\n";
return false;
}
int ret;
char buf[1024] = {0};
flock(fd, LOCK_EX);
while((ret = gzread(gf, buf, 1024)) > 0) {
int len = write(fd, buf, ret);
if(len < 0) {
std::cerr << "get gzip data failed\n";
flock(fd, LOCK_UN);
gzclose(gf);
close(fd);
return false;
}
}
flock(fd, LOCK_UN);
gzclose(gf);
close(fd);
unlink(gzip.c_str());
return true;
}
bool GetNormalFile(std::string &name, std::string &body) {
int64_t fsize = bf::file_size(name);
body.resize(fsize);
int fd = open(name.c_str(), O_RDONLY);
if(fd < 0) {
std::cerr << "open file " << name << " failed\n";
return false;
}
flock(fd, LOCK_SH);
int ret = read(fd, &body[0], fsize);
flock(fd, LOCK_UN);
if(ret != fsize) {
std::cerr << "get file " << name << " body error\n";
close(fd);
return false;
}
close(fd);
return true;
}
public:
CompressStore() {
pthread_rwlock_init(&_rwlock, nullptr);
if(!bf::exists(GZIPFILE_PATH)) {
bf::create_directory(GZIPFILE_PATH);
}
}
~CompressStore() {
pthread_rwlock_destroy(&_rwlock);
}
//向外提供獲取文件列表功能
bool GetFileList(std::vector<std::string> &list) {
pthread_rwlock_rdlock(&_rwlock);
for(auto i : _file_list) {
list.push_back(i.first);
}
pthread_rwlock_unlock(&_rwlock);
return true;
}
//通過文件名稱,獲取文件對應的壓縮包名稱
bool GetFileGzip(std::string &file, std::string &gzip) {
pthread_rwlock_rdlock(&_rwlock);
auto it = _file_list.find(file);
if(it == _file_list.end()){
pthread_rwlock_unlock(&_rwlock);
return false;
}
gzip = it->second;
pthread_rwlock_unlock(&_rwlock);
return true;
}
//向外提供獲取文件數據功能
bool GetFileData(std::string &file, std::string &body) {
if(bf::exists(file)) {
//1. 非壓縮文件數據獲取
GetNormalFile(file, body);
}else{
//2. 壓縮文件數據獲取
//獲取壓縮文件的數據需要先將壓縮文件解壓
std::string gzip;
GetFileGzip(file, gzip);
UnCompressFile(gzip, file);
GetNormalFile(file, body);
}
}
bool AddFileRecord(const std::string &file, const std::string &gzip) {
pthread_rwlock_wrlock(&_rwlock);
_file_list[file] = gzip;
pthread_rwlock_unlock(&_rwlock);
}
//文件存儲
bool SetFileData(const std::string &file, const std::string &body, const int64_t offset) {
int fd = open(file.c_str(), O_CREAT|O_WRONLY, 0664);
if(fd < 0) {
std::cerr << "open file " << file << "error\n";
return false;
}
flock(fd, LOCK_EX);
lseek(fd, offset, SEEK_SET);
int ret = write(fd, &body[0], body.size());
if(ret < 0) {
std::cerr << "store file " << file << " data error\n";
flock(fd, LOCK_UN);
close(fd);
return false;
}
flock(fd, LOCK_UN);
close(fd);
AddFileRecord(file, "");
return true;
}
//因爲壓縮存儲的流程是死循環,因此需要啓動線程來完成,下面的函數就是線程入口函數
bool LowHeatFileStore() { //對熱度低的文件進行壓縮存儲
//1. 獲取記錄信息
GetListRecord();
while(1){
//2. 目錄檢測,文件壓縮存儲
//2.1. 獲取list目錄下文件名稱
//2.2. 判斷文件是否需要壓縮存儲
//2.3. 對文件進行壓縮存儲
DirectoryCheck();
//3. 存儲記錄信息
SetListRecord();
sleep(3);
}
}
};