Qt：可限制最大內存的文件夾的動態讀、寫操作，支持單個超大文件和文件夾

一個用來對包含大量數據的文件夾進行動態的內存讀寫的類

默認單次讀寫最大文件夾的內存500MB，單個文件最大500MB，最大內容佔用1GB

fileoperator.h

#ifndef FILEOPERATOR_H
#define FILEOPERATOR_H
#include "libserver_global.h"
#include <QObject>
#include <QFileInfo>
#include <QDebug>

#define IntType qint64
#define IntSize sizeof(IntType)
const IntType Value_Folder = 0x0a0a0a01;
const IntType Value_BigFile = 0x0a0a0a02;
struct  ST_NewFile
{
	IntType		iNameStart;
	IntType		iNameLength;
	IntType		iDataStart;
	IntType		iDataLength;
	ST_NewFile * const parent;
	QByteArray  *d;
	ST_NewFile(ST_NewFile*parent_) :parent(parent_), iNameStart(-1), iNameLength(0), iDataStart(-1), iDataLength(0){ d = parent ? parent->d : NULL; }

	inline  ST_NewFile*root(){ if (parent)return parent->root(); else return this; }
	inline	virtual QString filePath(){
		if (!parent) return QString();
		return parent->filePath() + "/" + QString::fromLocal8Bit(name());
	}
	inline  virtual	IntType size(){ return iDataLength; }
	inline  QByteArray name();
	static QByteArray ReadFileData(QString sPath){
		QByteArray ret;
		if (QFile::exists(sPath))
		{
			QFile file(sPath);
			file.open(QIODevice::ReadOnly);
			ret = file.readAll();
			file.close();
		}
		return ret;
	}
	static bool WriteFile(QString sPath, QByteArray data, QIODevice::OpenMode iMode){
		QFile file(sPath);
		bool bRet = file.open(iMode);
		file.write(data);
		file.close();
		return bRet;
	}
};

struct LIBSERVER_EXPORT ST_NewFolder : public ST_NewFile
{
	QList<ST_NewFile> files;
	QList<ST_NewFolder*> folders;
	
	ST_NewFolder(ST_NewFile* parent_) :ST_NewFile(parent_) {}
	~ST_NewFolder();

	inline virtual QString filePath(){
		if (parent) return parent->filePath() + "/" + QString::fromLocal8Bit(name());
		else return QString::fromLocal8Bit(name());
	}
	virtual IntType size();
	ST_NewFolder* copy(QByteArray* = NULL);//完全拷貝
	IntType rootSize(){ return d->size(); }
	//
	bool   isEmpty();

	static ST_NewFolder* CreateRootFolder(){ ST_NewFolder*stRootFolder = new ST_NewFolder(NULL); stRootFolder->d = new QByteArray(); return stRootFolder; }
	static ST_NewFolder* CreateIndex(char*pD);
private:
	static bool CreateIndex(char*&pD, IntType &iIndex,ST_NewFolder*& stFolder);
};

/*
* 文件/目錄讀寫類
* 分層讀取
*/
class FileOperatorPrivate;
class  LIBSERVER_EXPORT FileOperator : public QObject
{
	Q_OBJECT
	Q_DECLARE_PRIVATE(FileOperator)
	Q_PROPERTY(IntType maxMemorySize READ maxMemorySize WRITE setMaxMemorySize);
	Q_PROPERTY(IntType maxFileSize READ maxFileSize WRITE setMaxFileSize);
	Q_PROPERTY(QString rootDir READ	rootDir WRITE setRootDir);
	Q_PROPERTY(QString newRootDir READ newRootDir WRITE	setNewRootDir);
	Q_PROPERTY(QString filePath READ filePath WRITE setFilePath);
	Q_PROPERTY(DataType dataType READ dataType);
public:
	enum DataType{
		Data_Bad = 0,
		Data_File,
		Data_Folder
	};
	Q_ENUMS(DataType)
	
	IntType maxMemorySize();
	void setMaxMemorySize(IntType iSize);
	IntType maxFileSize();
	void setMaxFileSize(IntType iSize);
	QString rootDir();
	void setRootDir(QString sDir);
	QString newRootDir();
	void setNewRootDir(QString sNewDir);
	QString filePath();
	void setFilePath(QString sFilePath);
	DataType dataType();
	
public:
	FileOperator(QString sDir, QString sNewPath, QObject *parent);
	//FileOperator(QString sFilePath, QString sNewPath, QObject *parent);
	FileOperator(QObject*parent);
	~FileOperator();
	//獲取文件夾信息
	//如目錄無效，返回NULL
	ST_NewFolder* getDirInfo(QString sDir);
	//
	bool	isVaild();
	bool	readDone();
	QByteArray read();
	bool	write(QByteArray);
	
};

#endif // FILEOPERATOR_H

fileoperator_p.h

#ifndef FILEOPERATORPRIVATE_H
#define FILEOPERATORPRIVATE_H

#include <QObject>
#if QT_VERSION <= QT_VERSION_CHECK(5,0,0)
#include <private/qobject_p.h>
#else
#include <QtCore/private/qobject_p.h>
#endif
#include "fileoperator.h"
#define BigFileMode_Create 0
#define BigFileMode_Append 1
class  FileOperatorPrivate : public QObjectPrivate
{
	Q_DECLARE_PUBLIC(FileOperator)

public:
	FileOperatorPrivate(int version = QObjectPrivateVersion);
	~FileOperatorPrivate();
	struct ST_BigFile{
		ST_BigFile(QString sPath_) :sPath(sPath_), iAllSize(0), iReadSize(0){
			iAllSize = QFileInfo(sPath).size();
		}
		QString sPath;
		QString sPath2;
		IntType	iAllSize;
		IntType  iReadSize;
	};
	void	getDirInfo(QString sDir, ST_NewFolder*& stFolder);
	void	read(ST_NewFolder*&);
	QByteArray	readBigFile();
	bool    write(char*&pD, QString sCurPath = QString());
	bool	writeFile(char*&pD);
	inline  void appendInt64(QByteArray &d, const IntType&iInt64){ d.append((char*)&iInt64, IntSize); }
	inline  IntType appendData(QByteArray&d, const QByteArray &data);
	//inline  void appendFile(QByteArray&d, const QString &sPath){ appendData(d, sPath.toLocal8Bit()); appendData(d, ST_File::ReadFileData(sPath)); }
	inline  IntType  getInt(char*&pD){ IntType iRet = *(IntType*)pD; pD += IntSize; return iRet; }
	inline  QByteArray getData(char*&pD){ IntType iSize = getInt(pD); QByteArray ret(pD, iSize); pD += iSize; return ret; }
public:
	IntType maxMemorySize;
	IntType maxFileSize;
	QString rootDir;
	QString newRootDir;
	QString filePath;
	FileOperator::DataType dataType;
	ST_NewFolder *rootFolder;
	ST_NewFolder *curReadFolder;
	QList<ST_BigFile> bigFiles;
private:
	
};

#endif // FILEOPERATORPRIVATE_H

fileoperator.cpp

#include "fileoperator.h"
#include "fileoperator_p.h"
#include <QDir>
//const int Value_Head = 0x01020304;
const int Value_500MB = 1024 * 1024 * 500;
const int Value_50MB = 1024 * 1024 * 50;
const int Value_GB = 1024 * 1024 * 1024;
#define DefaultMaxSize Value_500MB
//#define MaxMemorySize Value_50MB
//////////////////////////////////////////////////////////////////////////

ST_NewFolder::~ST_NewFolder()
{
	if (!parent && d)
		delete d; 
	qDeleteAll(folders);
	folders.clear();
}
ST_NewFolder* ST_NewFolder::copy(QByteArray* d_)
{
	ST_NewFolder* stRet = new ST_NewFolder(parent);
	if (!d_)
	{
		stRet->d = new QByteArray;
		*stRet->d = *d;
	}
	else
	{
		stRet->d = d_;
	}
	for (auto iter = files.begin(); iter != files.end(); iter++)
	{
		ST_NewFile stFile(stRet);
		stFile.iNameStart = (*iter).iNameStart;
		stFile.iNameLength = (*iter).iNameLength;
		stFile.iDataStart = (*iter).iDataStart;
		stFile.iDataLength = (*iter).iDataLength;
		stRet->files.append(stFile);
	}
	for (auto iter = folders.begin(); iter != folders.end();iter++)
		stRet->folders.append((*iter)->copy(stRet->d));
	return stRet;
}

IntType ST_NewFolder::size()
{
	IntType iRet = 0;
#if 0
	iRet = iDataLength;
#else
	for (auto iter = files.begin(); iter != files.end(); iter++)
		iRet += iter->iDataLength;
	for (auto iter = folders.begin(); iter != folders.end(); iter++)
		iRet += (*iter)->size();
#endif
	return iRet;
}

bool  ST_NewFolder::isEmpty()
{
	if (!files.isEmpty())
		return false;
	for (auto iter = folders.begin(); iter != folders.end();iter++)
		if (!(*iter)->isEmpty()) return false;
	return true;
}

ST_NewFolder* ST_NewFolder::CreateIndex(char*pD)
{
	return NULL;
	// TODO: 
// 	if (LibSvr::GetInt(pD) != Value_Folder)
// 		return NULL;
// 	ST_NewFolder* stFolder = NULL;
// 	IntType iIndex = IntSize;
// 	bool b = CreateIndex(pD, iIndex, stFolder);
// 	return stFolder;
}

bool ST_NewFolder::CreateIndex(char*&pD, IntType&iIndex, ST_NewFolder*& stFolder)
{
	// TODO: BUG filename 
	return false;
// 	static auto _GetInt = [&]()->int{
// 		iIndex += IntSize;
// 		return LibSvr::GetInt(pD);
// 	};
// 	static auto _GetData = [&]()->QByteArray{
// 		iIndex += IntSize;
// 		QByteArray ret = LibSvr::GetData(pD);
// 		iIndex += ret.size();
// 		return LibSvr::GetData(pD);
// 	};
// 	if (!stFolder)
// 	{
// 		stFolder = ST_NewFolder::CreateRootFolder();
// 	}
// 
// 	IntType iFolderNameLength = _GetInt();
// 	stFolder->iNameStart = iIndex;
// 	stFolder->iNameLength = iFolderNameLength;
// 	pD += iFolderNameLength;//跳過名字
// 	iIndex += iFolderNameLength;
// 	stFolder->iDataStart = iIndex;
// 	IntType iFileSize = _GetInt();
// 	for (int i = 0; i < _GetInt(); i++)
// 	{
// 		IntType iFileNameLength = _GetInt();
// 		ST_NewFile stFile(stFolder);
// 		stFile.iNameStart = iIndex;
// 		stFile.iNameLength = iFileSize;
// 		//跳過名字
// 		pD += iFileSize;
// 		iIndex += iFileSize;
// 		IntType iFileDataLength = _GetInt();
// 		stFile.iDataStart = iIndex;
// 		stFile.iDataLength = iFileDataLength;
// 		//跳過內容
// 		pD += iFileDataLength;
// 		iIndex += iFileDataLength;
// 	}
// 	IntType iFolderSize = _GetInt();
// 	for (int i = 0; i < iFolderSize; i++)
// 	{
// 		ST_NewFolder* stNewFolder = new ST_NewFolder(stFolder);
// 		stFolder->folders.append(stNewFolder);
// 		CreateIndex(pD, iIndex, stNewFolder);
// 	}
// 	return true;
}

//////////////////////////////////////////////////////////////////////////
IntType FileOperator::maxMemorySize()
{
	Q_D(FileOperator);
	return d->maxMemorySize;
}

void FileOperator::setMaxMemorySize(IntType iSize)
{
	Q_D(FileOperator);
	d->maxMemorySize = iSize;
}

IntType FileOperator::maxFileSize()
{
	Q_D(FileOperator);
	return d->maxFileSize;
}

void FileOperator::setMaxFileSize(IntType iSize)
{
	Q_D(FileOperator);
	d->maxFileSize = iSize;
}

QString FileOperator::rootDir()
{
	Q_D(FileOperator);
	return d->rootDir;
}

void FileOperator::setRootDir(QString sDir)
{
	Q_D(FileOperator);
	d->rootDir = sDir;
}

QString FileOperator::newRootDir()
{
	Q_D(FileOperator);
	return d->newRootDir;
}

void FileOperator::setNewRootDir(QString sNewDir)
{
	Q_D(FileOperator);
	d->newRootDir = sNewDir;
}

QString FileOperator::filePath()
{
	Q_D(FileOperator);
	return d->filePath;
}

QByteArray ST_NewFile::name()
{
	return QByteArray(d->data() + iNameStart, iNameLength); 
}

void FileOperator::setFilePath(QString sFilePath)
{
	Q_D(FileOperator);
	d->filePath = sFilePath;
}

FileOperator::DataType FileOperator::dataType()
{
	Q_D(FileOperator);
	return d->dataType;
}

FileOperator::FileOperator(QString sDir, QString sNewPath, QObject *parent)
	: QObject(*new FileOperatorPrivate, parent)
{
	Q_D(FileOperator);
	d->rootDir = sDir;
	d->newRootDir = sNewPath;
	d->getDirInfo(d->rootDir, d->rootFolder);
}

FileOperator::FileOperator(QObject*parent)
	:QObject(*new FileOperatorPrivate, parent)
{

}

FileOperator::~FileOperator()
{
	
}

ST_NewFolder *FileOperator::getDirInfo(QString sDir)
{
	Q_D(FileOperator);
	if (QDir().exists(sDir))
	{
		ST_NewFolder *stRet = NULL;
		d->getDirInfo(sDir, stRet);
		return stRet;
	}
	return NULL;
}

bool FileOperator::isVaild()
{
	Q_D(FileOperator);
	return QDir(d->rootDir).exists() ? true : false;
}

bool FileOperator::readDone()
{
	Q_D(FileOperator);
	if ((!d->rootFolder || d->rootFolder->isEmpty()) && d->bigFiles.isEmpty())
		return true;
	else
		return false;
}

QByteArray FileOperator::read()
{
	Q_D(FileOperator);
	ST_NewFolder *stFolder = ST_NewFolder::CreateRootFolder();
	QByteArray ret;
	if (!d->rootFolder->isEmpty())
	{
		d->curReadFolder = d->rootFolder;
		d->read(stFolder);
		d->appendInt64(ret, Value_Folder);
	}
	else
	{
		QByteArray data = d->readBigFile();
		data.insert(0,(char*)&Value_BigFile, IntSize);
		return data;
	}
	//更改路徑
	QByteArray dNewRootPath = d->newRootDir.toLocal8Bit();
	if (!dNewRootPath.endsWith('/') && !dNewRootPath.isEmpty())
		dNewRootPath.append('/');
	int iInsertPathSize = dNewRootPath.size();
	
	IntType iCurrentPathSize = *(IntType*)stFolder->d->data();
	ret.resize(stFolder->d->size() + iInsertPathSize + IntSize);
	IntType iTrueParhSize = iInsertPathSize + iCurrentPathSize;
	//複製新的路徑
	memcpy(ret.data() + IntSize, &iTrueParhSize, IntSize);//路徑長度
	memcpy(ret.data() + IntSize * 2, dNewRootPath.data(), iInsertPathSize);//路徑1
	memcpy(ret.data() + IntSize * 2 + iInsertPathSize, stFolder->d->data() + IntSize, iCurrentPathSize);//路徑2
	//複製數據
	memcpy(ret.data() + IntSize * 2 + iInsertPathSize + iCurrentPathSize,
		stFolder->d->data() + IntSize + iCurrentPathSize,
		stFolder->d->size() - IntSize - iCurrentPathSize);

	delete stFolder;
	return ret;
}

bool FileOperator::write(QByteArray data)
{
	Q_D(FileOperator);
	IntType iType = *(IntType*)data.data();
	char *pD = data.data() + IntSize;
	if (iType == Value_Folder)
		return d->write(pD);
	else if (iType == Value_BigFile)
		return d->writeFile(pD);
	else
		return false;
}
//////////////////////////////////////////////////////////////////////////

//////////////////////////////////////////////////////////////////////////
FileOperatorPrivate::FileOperatorPrivate(int version /* = QObjectPrivateVersion):QObjectPrivate(version */)
	:QObjectPrivate(version),
	maxMemorySize(DefaultMaxSize),
	maxFileSize(DefaultMaxSize),
	rootFolder(NULL),
	curReadFolder(NULL)
{
// 	ST_NewFolder *stF = NULL;
// 	getDirInfo("D:\\program\\lib", rootFolder);
// 	qint64 iSize = rootFolder->rootSize();
// 	curReadFolder = rootFolder;
// 	ST_NewFolder *stReadF = ST_NewFolder::CreateRootFolder();
// 	read(stReadF);
// 	char*pD = stReadF->d->data();
// 	write(pD,"./haha");
}

FileOperatorPrivate::~FileOperatorPrivate()
{
	if (rootFolder)
		delete rootFolder;

}

void FileOperatorPrivate::getDirInfo(QString sDir, ST_NewFolder*& stFolder)
{
	QDir dir(sDir);
	QByteArray dDir = dir.dirName().toLocal8Bit();
	if (!stFolder)
	{
		stFolder = ST_NewFolder::CreateRootFolder();
		dDir = dir.absolutePath().toLocal8Bit();
	}
	QByteArray&d = *stFolder->d;
	
	stFolder->iNameStart = d.size() + IntSize;
	stFolder->iNameLength = appendData(d, dDir);
	stFolder->iDataStart = stFolder->iNameStart + stFolder->iNameLength;
	//qDebug() << stFolder->filePath();
	auto infos = dir.entryInfoList(QDir::Files | QDir::NoDotAndDotDot);
	appendInt64(d, infos.size());
	for (int i = 0; i < infos.size(); i++)
	{
		QFileInfo &info = infos[i];
		ST_NewFile stFile(stFolder);
		stFile.iNameStart = d.size() + IntSize;
		QByteArray dName = info.fileName().toLocal8Bit();
		stFile.iNameLength = appendData(d, dName);
		stFile.iDataLength = info.size();
		stFolder->files.append(stFile);
		//qDebug() << stFile.filePath() << "\tsize:"<<stFile.iDataLength;
	}

	infos = dir.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot);
	appendInt64(d, infos.size());
	for (int i = 0; i < infos.size(); i++)
	{
		QFileInfo &info = infos[i];
		ST_NewFolder *stNewFolder = new ST_NewFolder(stFolder);
		stFolder->folders.append(stNewFolder);
		getDirInfo(info.absoluteFilePath(), stNewFolder);
	}
	stFolder->iDataLength = d.size() - stFolder->iDataStart;
}

void FileOperatorPrivate::read(ST_NewFolder*& stFolder)
{
	QByteArray &d = *stFolder->d;
	stFolder->iNameStart = d.size() + IntSize;
	QByteArray dFolderName = curReadFolder->name();
	if (dFolderName.contains('/'))
	{
		auto bas = dFolderName.split('/');
		if (bas.last().isEmpty())
			bas.takeLast();
		dFolderName = bas.last();
	}
	stFolder->iNameLength = appendData(d, dFolderName);
	IntType iFileSize = 0;
	int iFileSizeIndex = d.size();
	appendInt64(d, curReadFolder->files.size());
	for (auto iter = curReadFolder->files.begin(); iter != curReadFolder->files.end(); iter++)
	{
		auto& stFile = *iter;
		if (stFile.size() < maxFileSize)
		{
			iFileSize++;
			ST_NewFile stNewFile(stFolder);
			stNewFile.iNameStart = d.size() + IntSize;
			stNewFile.iNameLength = appendData(d, stFile.name());
			stNewFile.iDataStart = d.size() + IntSize;
			stNewFile.iDataLength = appendData(d, ST_NewFile::ReadFileData(stFile.filePath()));
			if (!QFile::exists(stFile.filePath()))
				qDebug() << stFile.filePath();
			stFolder->files.append(stNewFile);
			//qDebug() << "read:" << stNewFile.filePath() << "\t size:" << stNewFile.iDataLength;
		}
		else
		{
			// 移到最後單獨讀取
			ST_BigFile stBigFile(stFile.filePath());
			stBigFile.sPath2 = stFolder->filePath();
			bigFiles.append(stBigFile);
		}
		curReadFolder->files.erase(iter);
		if (stFolder->rootSize() >= maxMemorySize)
			break;
	}
	memcpy(d.data() + iFileSizeIndex, &iFileSize, IntSize);
	IntType iFolderSize = 0;
	int    iFolderSizeIndex = d.size();
	appendInt64(d, curReadFolder->folders.size());
	ST_NewFolder *oldFolder = curReadFolder;
	for (auto iter = oldFolder->folders.begin(); iter != oldFolder->folders.end(); iter++)
	{
		ST_NewFolder *stNewFolder = new ST_NewFolder(stFolder);
		curReadFolder = *iter;
		read(stNewFolder);
		iFolderSize++;
		if (curReadFolder->isEmpty())
		{
			delete *iter;
			oldFolder->folders.erase(iter);
		}
		if (stFolder->rootSize() >= maxMemorySize)
			break;
	}
	memcpy(d.data() + iFolderSizeIndex, &iFolderSize, IntSize);
	curReadFolder = oldFolder;
}

QByteArray FileOperatorPrivate::readBigFile()
{
	QByteArray ret;
	if (bigFiles.isEmpty())
	{
		appendInt64(ret, 0);
		return ret;
	}
	appendInt64(ret, bigFiles.size());
	IntType iBigFileSize = 0;
	for (auto iter = bigFiles.begin(); iter != bigFiles.end(); iter++)
	{
		iBigFileSize++;
		ST_BigFile & bf = *iter;
		IntType iWaitSize = bf.iAllSize - bf.iReadSize;
		IntType iReadSize = iWaitSize;
		if (iReadSize + ret.size() > maxMemorySize)
			iReadSize = maxMemorySize - ret.size();
		//放到這裏，防止路徑佔用過多字節
		QString sPath = bf.sPath2 + "/" + QFileInfo(bf.sPath).fileName();
		if (!newRootDir.isEmpty())
			sPath = newRootDir + "/" + sPath;
		appendData(ret, sPath.toLocal8Bit());//文件路徑
		appendInt64(ret, bf.iReadSize == 0 ? BigFileMode_Create : BigFileMode_Append);//文件模式
		QFile file(bf.sPath);
		file.open(QIODevice::ReadOnly);
		file.seek(bf.iReadSize);
		appendData(ret,file.read(iReadSize));
		file.close();
		bf.iReadSize += iReadSize;
		if (bf.iReadSize >= bf.iAllSize)
			bigFiles.erase(iter);
		if (ret.size() >= maxMemorySize)
			break;	
	}
	memcpy(ret.data(), &iBigFileSize, IntSize);
	return ret;
}

bool FileOperatorPrivate::write(char*&pD, QString sRootPath)
{
	QString sTruePath = QString::fromLocal8Bit(getData(pD)) + "/";
	if (!sRootPath.isEmpty())
		sTruePath = sRootPath + "/" + sTruePath;
	if (!QDir().exists(sTruePath))
		QDir().mkpath(sTruePath);
	IntType iFileSize = getInt(pD);
	for (int i = 0; i < iFileSize; i++)
	{
		QString sFileName = QString("./")+sTruePath + QString::fromLocal8Bit(getData(pD));
		QByteArray d = getData(pD);
		QFile file(sFileName);
		bool b = file.open(QIODevice::WriteOnly);
		file.write(d);
		file.close();
	}
	IntType iFolderSize = getInt(pD);
	for (int i = 0; i < iFolderSize; i++)
	{
		write(pD, sTruePath);
	}
	return false;
}

bool FileOperatorPrivate::writeFile(char*&pD)
{
	IntType iFileSize = getInt(pD);
	for (int i = 0; i < iFileSize; i++)
	{
		QString sFilePath = QString::fromLocal8Bit(getData(pD));
		IntType iMode = getInt(pD);
		QByteArray data = getData(pD);
		//qDebug() << "write big:" << sFilePath;
		if (iMode == BigFileMode_Create)
		{
			ST_NewFile::WriteFile(sFilePath, data, QIODevice::WriteOnly);
		}
		else if (iMode == BigFileMode_Append)
		{
			ST_NewFile::WriteFile(sFilePath, data, QIODevice::WriteOnly | QIODevice::Append);
		}
	}
	return true;
}
IntType FileOperatorPrivate::appendData(QByteArray&d, const QByteArray &data)
{
	IntType iRet = data.size();
	appendInt64(d, iRet);
	d.append(data);
	return iRet;
}