Codes

server.cpp

#include "httplib.h"
#include <boost/filesystem.hpp>
#include <iostream>
#include <sstream>
#include <fstream>

#if 0
void HelloWorld(const httplib::Request &req, httplib::Response &rsp)
{
	//text/html 顯示在瀏覽器上的
	rsp.set_content("<html><h1>Hello World</h1></html>", "text/html");
}

int main()
{
	httplib::Server srv;
	//設置相對根目錄
	srv.set_base_dir("./www");
	//當客戶端發送get請求,該如何去處理
	//    192.168.43.85:9000/
	srv.Get("/", HelloWorld);
	srv.listen("0.0.0.0", 9000);
	return 0;

}
#endif

#define SERVER_BASE_DIR "www"
#define SERVER_ADDR "0.0.0.0"
#define SERVER_PORT 9000
#define SERVER_BACKUP_DIR SERVER_BASE_DIR"/list/"

using namespace httplib;
namespace bf = boost::filesystem;

class CloudServer{
private:
	httplib::Server srv;
public:
	CloudServer(){
		bf::path base_path(SERVER_BASE_DIR);
		//如果文件不存在
		if (!bf::exists(base_path)){
			//創建目錄
			bf::create_directory(base_path);
		}
		bf::path list_path(SERVER_BACKUP_DIR);
		if (!bf::exists(list_path)){
			bf::create_directory(list_path);
		}
	}
	bool Start(){
		srv.set_base_dir(SERVER_BASE_DIR);
		srv.Get("/(list(/){0,1}){0,1}", GetFileList);
		srv.Get("/list/(.*)", GetFileData);
		srv.Put("/list/(.*)", PutFileData);
		srv.listen("SERVER_ADDR", SERVER_PORT);
		return true;
	}
private:
	static void PutFileData(const Request &req, httplib::Response &rsp)
	{
		std::cout << "backup file " << req.path << "\n";
		if (!req.has_header("Range")){
			rsp.status = 400;
			return;
		}

			std::string range = req.get_header_value("Range");
			int64_t range_start;
			if (RangeParse(range, range_start) == false){
				rsp.status = 400;
				return;
			
		}
			std::string real = SERVER_BASE_DIR + req.path;
			std::ofstream file(real, std::ios::binary | std::ios::trunc);
			if (!file.is_open()){
				std::cerr << "open file " << real << "error\n";
				rsp.status = 500;
				return;
			}
			file.seekp(range_start, std::ios::beg);
			file.write(&req.body[0], req.body.size());
			if (!file.good()){
				std::cerr << "file write body error\n";
				rsp.status = 500;
				return;
			}
			file.close();
		return;
	}

	static bool RangeParse(std::string &range, int64_t &start){

		//Range: bytes=start-end
		size_t pos1 = range.find("=");
		size_t pos2 = range.find("=");
		if (pos1 == std::string::npos || pos2 == std::string::npos){
			std::cerr << "range:[" << range << "] format error\n";
			return false;
		}
		std::stringstream rs; 
		rs << range.substr(pos1 + 1, pos2 - pos1 - 1);
		rs >> start;
		return true;
	}
	//文件列表信息獲取,定義成靜態就是避免了this指針
	static void GetFileList(const Request &req, httplib::Response &rsp){
		bf::path list(SERVER_BACKUP_DIR);
		bf::directory_iterator item_begin(list);
		bf::directory_iterator item_end;

		std::string body;
		body = "<html><body><ol><hr />";



		for (; item_begin != item_end; ++item_begin){
			//如果不是目錄
			if (bf::is_directory(item_begin->status())){
				continue;
			}
			std::string file = item_begin->path().filename().string();
			std::string uri = "/list/" + file;
			body += "<h4><li>";
			body += "<a href='";
			body += uri;
			body += "'>";
			body += file;
			body += "</a>";
			body += "</li></h4>";
			//std::string file = item_begin->path().string();
			//std::cerr << "file:" << file << std::endl;
		}
		body += "<hr /></ol></body></html>";
		rsp.set_content(&body[0], "text/html");
		return;
	}
	//獲取文件數據(文件下載)
	static void GetFileData(const Request &req, httplib::Response &rsp){
		//req.path = "/list/a.txt";
		std::string file = SERVER_BACKUP_DIR + req.path;
		//文件不存在,page not found
		if (!bf::exists(file)){
			std::cerr << "file " << file << "is not exists\n";
			rsp.status = 404;
			return;
		}
		//文件存在
		//ifile = inputfile
		std::ifstream ifile(file, std::ios::binary);
		if (!ifile.is_open()){
			std::cerr << "open file " << file << "error\n";
			rsp.status = 500; //服務器內部錯誤
			return;
		}
		std::string body;
		//獲取文件大小
		int64_t fsize = bf::file_size(file);
		body.resize(fsize);
		ifile.read(&body[0], fsize);
		if (ifile.good()){
			std::cerr << "read file " << file << "body error\n";
			rsp.status = 500;
			return;
		}
		//正文只能給一次,文件不能太大
		rsp.set_content(body, "text/plain"); //plain文件下載
	}
	//解決文件上傳
	static void BackuoFile(const Request &req, httplib::Response &rsp){

	}


};

int main()
{
	CloudServer srv;
	srv.Start();
	return 0;
}

CloudClient.hpp

#define _D_SCL_SECURE_NO_WARNINGS 1
#ifndef __M_CLOUD_H__
#define __M_CLOUD_H__

#include <fstream>
#include <sstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>
#include <thread>
#include "httplib.h"

#define SERVER_IP "192.168.122.132"
#define SERVER_PORT 9000
#define CLIENT_BACKUP_DIR ".\\backup"
#define CLIENT_BACKUP_INFO_FILE ".\\back.list"
#define RANGE_MAX_SIZE (10 << 20) //10MB
#define BACKUP_URI "/list/"

namespace bf = boost::filesystem;

class ThrBackUp
{
private:
	std::string _file;
	int64_t _range_start;
	int64_t _range_len;
public:
	bool _res;
public:
	ThrBackUp(const std::string &file, int64_t start, int64_t len) :_res(true),
	_file(file), _range_start(start), _range_len(len) {}
	void Start(){
		//獲取文件的range分塊數據
		std::stringstream ss;
		ss << "backup file" << _file << "range:[" << _range_start << "-" << _range_len << "\n";
		std::cout << ss.str();
		std::ifstream path(_file, std::ios::binary);
		if (!path.is_open()){
			std::cerr << "range backup file " << _file << "failed!\n";
			_res = false;
			return;
		}
		//跳轉到range的起始位置
		path.seekg(_range_start, std::ios::beg);//跳轉
		std::string body;
		
		body.resize(_range_len);
		//讀取文件中range分塊的文件數據
		path.read(&body[0], _range_len);
		if (!path.good()){
			std::cerr << "read file" << _file << " range data failed\n";
			_res = false;
			return;
		}
		path.close();
		//上傳range數據
		bf::path name(_file);
		//組織上傳的uri路徑  method url version
		//PUT /list/filename HTTP/1.1
		std::string url = BACKUP_URI + name.filename().string();//native  不要雙引號
		//實例化一個httplib的客戶端對象
		httplib::Client cli(SERVER_IP, SERVER_PORT);
		//定義HTTP請求頭信息
		httplib::Headers hdr;
		hdr.insert(std::make_pair("Content-Length", std::to_string(_range_len)));

		std::stringstream tmp;
		tmp << "bytes=" << _range_start << "-" << _range_start + _range_len - 1;
		hdr.insert(std::make_pair("Range", tmp.str().c_str()));
		//通過實例化的Client向服務端發送PUT請求
		auto rsp = cli.Put(url.c_str(), hdr, body, "text/plain");
		if (rsp &&rsp->status != 200){
			_res = false;
		}
		else {
			std::stringstream ss;
			ss << "backup file" << _file << "] range:[" << _range_start << "-" << _range_len << "] backup success\n";
			std::cout << ss.str();
			return;
		}
		//return ;
	}
};
class CloudClient
{
public:
	CloudClient(){
		bf::path file(CLIENT_BACKUP_DIR);
		if (!bf::exists(file)){
			bf::create_directory(file);
		}
	}
private:
	std::unordered_map<std::string, std::string> _backup_list;
	//CloudClient(){}
private:
	
	//獲取備份信息
	bool GetBackupInfo()
	{
		//filename1 etag\n
		//filename2 etag\n
		//實例化path對象
		bf::path path(CLIENT_BACKUP_INFO_FILE);
		if (!bf::exists(path)){
			std::cerr << "list file" << path.string() << "is not exists\n";
			return false;
		}
		int64_t fsize = bf::file_size(path);
		if (fsize == 0){
			std::cerr << "have no backup info\n";
			return false;
		}
		std::string body;
		body.resize(fsize);
		std::ifstream file(CLIENT_BACKUP_INFO_FILE, std::ios::binary);
		if (!file.is_open()){
			std::cerr << "list file open error\n";
			return false;
		}
		file.read(&body[0], fsize);
		if (!file.good()){
			std::cerr << "read list file body error\n";
			return false;
		}
		file.close();
		std::vector<std::string> list;
		boost::split(list, body, boost::is_any_of("\n"));//切割後放到list中

		for (auto i : list){
			//filename2 etag
			size_t pos = i.find(" ");
			if (pos == std::string::npos)
				continue;
			std::string key = i.substr(0, pos);
			std::string val = i.substr(pos + 1);
			_backup_list[key] = val;
		}
		return true;
	}
	//將備份信息保存在指定文件中
	bool SetBackuoInfo()
	{
		std::string body;
		for (auto i : _backup_list){
			body += i.first + " " + i.second + "\n";
		}
		std::ofstream file(CLIENT_BACKUP_INFO_FILE, std::ios::binary);
		if (!file.is_open()){
			std::cerr << "open list file error\n";
			return false;
		}
		file.write(&body[0], body.size());
		if (!file.good()){
			std::cerr << "set backuo info error\n";
			return false;
		}
		file.close();
		return true;
	}
	//目錄監控
	bool BackupDirListen(const std::string &path)
	{
		bf::directory_iterator item_begin(path);
		bf::directory_iterator item_end;
		for (; item_begin != item_end; ++item_begin){
			// /list/abc/abc.txt bc.txt
			if (bf::is_directory(item_begin->status())){
				BackupDirListen(item_begin->path().string());
				continue;
			}
			if (FileIsNeedBackup(item_begin->path().string()) == false){
				continue;
			}
			std::cerr << "file:[" << item_begin->path().string() << "need backup\n";
			if (PutFileData(item_begin->path().string()) == false){
				continue;
			}
			//上傳成功後
			//記錄文件Etag信息
			AddBackInfo(item_begin->path().string());

		}
		return true;
	}

	//添加一個信息
	bool AddBackInfo(const std::string &file)
	{
		//etag = "mtime-fsize"
		std::string etag;
		if (GetFileEtag(file, etag) == false)
			return false;
		_backup_list[file] = etag;
	}
	static void thr_start(ThrBackUp *backuo_info){
		/*

		*/
		return;
	}

	//備份文件
	bool PutFileData(const std::string &file)
	{
		//按大小對文件內容進行分塊傳輸(10MB)
		//通過獲取分塊傳輸是否成功判斷整個文件是否上傳成功
		//選擇多線程處理
		//1.獲取文件大小
		int64_t fsize = bf::file_size(file);
		if (fsize <= 0){
			std::cerr << "file " << file << "unnecessary backup\n";
			return false;
		}
		//2.計算總共需要分多少塊,得到每塊大小以及起始位置
		//3.循環創建線程,在線程中上傳文件數據
		int count = (int)fsize / RANGE_MAX_SIZE;
		std::vector<ThrBackUp>thr_res;
		std::vector<std::thread> thr_list;
		std::cerr << "file:[" << file << "]fsize:[" << fsize << "] count:[" << count + 1 << "]\n";
		// fsize = 500;
		for (int i = 0; i <= count; ++i){
			int64_t range_start = i * RANGE_MAX_SIZE; //起始位置
			int64_t range_end = (i + 1) * RANGE_MAX_SIZE - 1;
			if (i == count-1){
				range_end += (fsize % RANGE_MAX_SIZE);
			}
			int64_t range_len = range_end - range_start + 1;
			ThrBackUp backup_info(file, range_start, range_len);
			std::cerr << "file:[" << file << "] range:[" << range_start << "-" << range_end << "]-" << range_len << "\n";
			thr_res.push_back(backup_info);
			thr_list.push_back(std::thread(thr_start, &thr_res[i]));
			//std::thread thr(thr_start, file, range_start, range_len, &thr_res[i]);

		}
		/*for (int i = 0; i <= count; ++i)
		{
			thr_list.push_back(std::thread(thr_start, &thr_res[i]));
		}*/

		//4.等待所有線程退出,判斷文件上傳結果
		bool ret = true;
		for (int i = 0; i <= count; ++i){
			thr_list[i].join();
			if (thr_res[i]._res == true)
				continue;
		}	
		ret = false;
		//5.上傳成功,則添加文件的備份信息記錄。 native() //不要雙引號
		if (ret == false){
			return false;
		}
			//AddBackInfo(file);
			std::cerr << "file:[" << file << "] backup success\n";
			return true;
	}
	
	bool FileIsNeedBackup(const std::string &file)
	{
		std::string etag;
		if (GetFileEtag(file, etag) == false)
			return false;
		auto it = _backup_list.find(file);
		if (it != _backup_list.end() && it->second == etag){
			//不需要備份
			return false;
		}
		
		return true;
	}
	bool GetFileEtag(const std::string &file, std::string &etag)
	{
		bf::path path(file);
		if (!bf::exists(path))
		{
			std::cerr << "get file" << file << "etag error\n";
			return false;
		}
		int64_t fsize = bf::file_size(path);
		int64_t mtime = bf::last_write_time(path);
		std::stringstream tmp;
		tmp << std::hex << fsize << "-" << std::hex << mtime;
		etag = tmp.str();
		return true;
	}
	static void thr_start(ThrBackUp *backup_info)
	{
		backup_info->Start();
		return;
	}
public:
	bool Start()
	{
		
		GetBackupInfo();//獲取備份信息
		while (1)
		{
			//對文件目錄進行監控
			BackupDirListen(CLIENT_BACKUP_DIR);
			SetBackuoInfo();
			Sleep(10);
		}
		return true;
	}
};
#endif

compress.hpp

#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
#include <unordered_map>
#include <vector>
#include <thread>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <zlib.h>
#include <pthread.h>
#include <sys/file.h>

//未壓縮文件存儲位置
#define UNGZIPFILE_PATH	    "www/list/"
//壓縮包存儲位置
#define GZIPFILE_PATH	    "www/zip/"
//存放文件信息(filename gzipfilename\n)
#define RECORD_FILE	    "record.list"
//熱度過期時間
#define HEAT_TIME	     2

namespace bf = boost::filesystem;

class CompressStore
{
  private:
    //用於保存文件列表
    std::unordered_map<std::string, std::string> _file_list;
    pthread_rwlock_t _rwlock;
  private:
    //1. 每次壓縮存儲線程啓動的時候,從文件中讀取列表信息
    bool GetListRecord() {
      //存儲列表信息格式如下:
      //filename gzipfilename\n
      bf::path name(RECORD_FILE);
      if(!bf::exists(name)) {
        std::cerr << "record file is not exists\n";
        return false;
      }
      std::ifstream file(RECORD_FILE, std::ios::binary);
      if(!file.is_open()){
        std::cerr << "open record file error\n";
        return false;
      }
      int64_t fsize = bf::file_size(name);
      std::string body;
      body.resize(fsize);
      file.read(&body[0], fsize);
      if(!file.good()){
        std::cerr << "record file body read error\n";
        return false;
      }
      file.close();

      std::vector<std::string> list;
      boost::split(list, body, boost::is_any_of("\n"));
      for(auto e : list) {
        //分割後的信息如下
        //filename gzipfilename
        size_t pos = e.find(" ");
        if(pos == std::string::npos) {
          continue;
        }
        std::string key = e.substr(0, pos);
        std::string val = e.substr(pos + 1);
        _file_list[key] = val;
      }
      return true;
    }
    //2. 每次壓縮存儲完畢,都要將列表信息,存儲到文件中
    bool SetListRecord() {
      std::stringstream tmp;
      for(auto i : _file_list) {
        tmp << i.first << " " << i.second << "\n";
      }
      std::ofstream file(RECORD_FILE, std::ios::binary | std::ios::trunc);
      if(!file.is_open()) {
        std::cerr << "record file open error\n";
        return false;
      }
      file.write(tmp.str().c_str(), tmp.str().size());
      if(!file.good()) {
        std::cerr << "record file write body error\n";
        return false;
      }
      file.close();
      return true;
    }
    //目錄檢測,獲取目錄中的文件名
    //  1.判斷文件是否需要壓縮存儲
    //  2.文件壓縮存儲
    bool DirectoryCheck() {
      if(!bf::exists(UNGZIPFILE_PATH)) {
        bf::create_directory(UNGZIPFILE_PATH);
      }
      std::string cmd = "bash gbk_utf8";
      FILE *fd = popen(cmd.c_str(),"r");
      pclose(fd);
      bf::directory_iterator item_begin(UNGZIPFILE_PATH);
      bf::directory_iterator item_end;

      for(; item_begin != item_end; ++item_begin) {
        if(bf::is_directory(item_begin->status())) {
          continue;
        }
        std::string name = item_begin->path().string();
        if(!IsNeedCompress(name)) {
          continue;
        }
        std::string gzip = GZIPFILE_PATH + item_begin->path().filename().string() + ".gz";
        if(CompressFile(name, gzip)) {
          std::cerr << "file:[" << name << "] store success\n";
          AddFileRecord(name, gzip);
        }else {
          std::cerr << "file:[" << name << "] store failed\n";
        }
      }
      return true;
    }
    //2.2. 判斷文件是否需要壓縮存儲
    bool IsNeedCompress(std::string &file) {
      //通過stat這個系統調用接口可以獲取文件相關信息,取最後一次訪問時間
      struct stat st;
      if(stat(file.c_str(), &st) < 0) {
        std::cerr << "get file:[" << file << "] stat error\n";
        return false;
      }
      time_t cur_time = time(nullptr);	//當前系統時間
      time_t acc_time = st.st_atime;
      if((cur_time - acc_time) < HEAT_TIME){
        return false;
      }
      return true;
    }
    //2.3. 對文件進行壓縮存儲
    bool CompressFile(std::string &file, std::string &gzip) {
      int fd = open(file.c_str(), O_RDONLY);
      if(fd < 0) {
        std::cerr << "com open file:[" << file << "] error\n";
        return false;
      }
      gzFile gf = gzopen(gzip.c_str(), "wb");
      if(gf == nullptr) {
        std::cerr << "com open gzip:[" << gzip<< "] error\n";
        return false;
      }
      int ret;
      char buf[1024];
      flock(fd, LOCK_SH);
      while((ret = read(fd, buf, 1024)) > 0) {
        gzwrite(gf, buf, ret);
      }
      flock(fd, LOCK_UN);
      close(fd);
      gzclose(gf);

      //壓縮完後,原文件就沒必要存在了,所以調用unlink刪除原文件
      //int unlink(const char *pathname);
      //如果文件正在被操作,那麼會報錯:EBUSY
      //EBUSY:The file pathname cannot be unlinked because it is being used by the system or another process; for example, it is a mount point or the NFS client software created it to represent an active but otherwise nameless inode ("NFS silly renamed").
      unlink(file.c_str());   //unlink 在刪除文件時,如果這個文件已經被打開且正在被操作,這個文件就無法刪除
      return true;
    }

    //對文件解壓縮
    bool UnCompressFile(std::string &gzip, std::string &file) {
      int fd = open(file.c_str(), O_CREAT | O_WRONLY, 0664);
      if(fd < 0) {
        std::cerr << "open file" << file << "failed\n";
        return false;
      }
      gzFile gf = gzopen(gzip.c_str(), "rb");
      if(gf == nullptr) {
        std::cerr << "open gzip " << gzip << "failed\n";
        return false;
      }
      int ret;
      char buf[1024] = {0};
      flock(fd, LOCK_EX);
      while((ret = gzread(gf, buf, 1024)) > 0) {
        int len = write(fd, buf, ret);
        if(len < 0) {
          std::cerr << "get gzip data failed\n";
          flock(fd, LOCK_UN);
          gzclose(gf);
          close(fd);
          return false;
        }
      }
      flock(fd, LOCK_UN);
      gzclose(gf);
      close(fd);
      unlink(gzip.c_str());
      return true;
    }

    bool GetNormalFile(std::string &name, std::string &body) {
      int64_t fsize = bf::file_size(name);
      body.resize(fsize);

      int fd = open(name.c_str(), O_RDONLY);
      if(fd < 0) {
        std::cerr << "open file " << name << " failed\n";
        return false;
      }

      flock(fd, LOCK_SH);
      int ret = read(fd, &body[0], fsize);
      flock(fd, LOCK_UN);
      if(ret != fsize) {
        std::cerr << "get file " << name << " body error\n";
        close(fd);
        return false;
      }
      close(fd);
      return true;
    }

  public:
    CompressStore() {
      pthread_rwlock_init(&_rwlock, nullptr);
      if(!bf::exists(GZIPFILE_PATH)) {
        bf::create_directory(GZIPFILE_PATH);
      }
    }
    ~CompressStore() {
      pthread_rwlock_destroy(&_rwlock);
    }
    //向外提供獲取文件列表功能
    bool GetFileList(std::vector<std::string> &list) {
      pthread_rwlock_rdlock(&_rwlock);
      for(auto i : _file_list) {
        list.push_back(i.first);
      }
      pthread_rwlock_unlock(&_rwlock);
      return true;
    }

    //通過文件名稱,獲取文件對應的壓縮包名稱
    bool GetFileGzip(std::string &file, std::string &gzip) {
      pthread_rwlock_rdlock(&_rwlock);
      auto it = _file_list.find(file);
      if(it == _file_list.end()){
        pthread_rwlock_unlock(&_rwlock);
        return false;
      }
      gzip = it->second;
      pthread_rwlock_unlock(&_rwlock);
      return true;
    }

    //向外提供獲取文件數據功能
    bool GetFileData(std::string &file, std::string &body) {
      if(bf::exists(file)) {
        //1. 非壓縮文件數據獲取
        GetNormalFile(file, body);
      }else{
        //2. 壓縮文件數據獲取
        //獲取壓縮文件的數據需要先將壓縮文件解壓
        std::string gzip;
        GetFileGzip(file, gzip);
        UnCompressFile(gzip, file);
        GetNormalFile(file, body);
      }
    }

    bool AddFileRecord(const std::string &file, const std::string &gzip) {
      pthread_rwlock_wrlock(&_rwlock);
      _file_list[file] = gzip; 
      pthread_rwlock_unlock(&_rwlock);
    }

    //文件存儲
    bool SetFileData(const std::string &file, const std::string &body, const int64_t offset) {
      int fd = open(file.c_str(), O_CREAT|O_WRONLY, 0664);
      if(fd < 0) {
        std::cerr << "open file " << file << "error\n";
        return false;
      }
      flock(fd, LOCK_EX);
      lseek(fd, offset, SEEK_SET);
      int ret = write(fd, &body[0], body.size());
      if(ret < 0) {
        std::cerr << "store file " << file << " data error\n";
        flock(fd, LOCK_UN);
        close(fd);
        return false;
      }
      flock(fd, LOCK_UN);
      close(fd);
      AddFileRecord(file, "");
      return true;
    }
    //因爲壓縮存儲的流程是死循環,因此需要啓動線程來完成,下面的函數就是線程入口函數
    bool LowHeatFileStore() {	//對熱度低的文件進行壓縮存儲
      //1. 獲取記錄信息
      GetListRecord();
      while(1){
        //2. 目錄檢測,文件壓縮存儲
        //2.1. 獲取list目錄下文件名稱
        //2.2. 判斷文件是否需要壓縮存儲
        //2.3. 對文件進行壓縮存儲
        DirectoryCheck();
        //3. 存儲記錄信息
        SetListRecord();
        sleep(3);
      }
    }
};
發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章