點雲讀取速度比較——QTextStream、C++文件流、C++文件映射

    最近研究了一下CC的點雲讀取類，速度簡直快到不行。

    後來發現CC就是簡單使用了QTextStream進行讀取。

    筆者之前研究過文件映射進行點雲讀取，速度也是非常快。內存映射之所以能達到這麼高的速度是因爲系統直接把整塊硬盤內存直接交由程序處理，省去了數據交換過程。

    那麼文件映射和QTextStream究竟誰快呢？

    筆者準備了一個1000W多的XYZ格式的點雲，點雲內部有6列，分別是XYZRGB，分別用三種方式讀寫，然後記錄了時間。

    下面先上程序：

    首先是文件映射的方式讀取，筆者就把讀取XYZ格式的點雲：

//PTX format:
//All the row is point cloud
//field width : unknown
int qScarlet_GLCloudEngin_ES2::ReadPointCloudFrom_XYZ()
{
    HANDLE hSrcFile = CreateFileA(m_FullPath.c_str(), GENERIC_READ, 0, NULL, OPEN_ALWAYS, 0, NULL);
    if (hSrcFile == INVALID_HANDLE_VALUE) return 0;
    LARGE_INTEGER tInt2;
    GetFileSizeEx(hSrcFile, &tInt2);
    __int64 dwRemainSize = tInt2.QuadPart;
    __int64 dwFileSize = dwRemainSize;
    HANDLE hSrcFileMapping = CreateFileMapping(hSrcFile, NULL, PAGE_READONLY, tInt2.HighPart, tInt2.LowPart, NULL);
    if (hSrcFileMapping == INVALID_HANDLE_VALUE)
    {
        return 0;
    }
    SYSTEM_INFO SysInfo;
    GetSystemInfo(&SysInfo);
    DWORD dwGran = SysInfo.dwAllocationGranularity;
    const int BUFFERBLOCKSIZE = dwGran * 1024;

    const int XYZ_FC = 3;
    const int XYZI_FC = 4;
    const int XYZRGB_FC = 6;
    const int XYZIRGB_FC = 7;
    bool AlreadySetFiledCount = false;//是否已經設置了數據寬度
    int usefulFiledCount = 0;

    int totalRows = 0;//文件總行數：
    int FieldIndex = 0;//每一個小數字的填充位置
    int   FieldCount = 0;//每一行中整數字位置，用來判定數據列數究竟是XYZARGB。
    double arrXYZ_RGB[XYZIRGB_FC];
    char  strLine[1024] = { 0 };

    double time = (double)cv::getTickCount();
    vector<CloudVertex> PointVec;//就怕這玩意溢出啊
    vector<CloudVertex>().swap(PointVec);//先進行清空
    while (dwRemainSize > 0)
    {
        DWORD dwBlock = dwRemainSize < BUFFERBLOCKSIZE ? dwRemainSize : BUFFERBLOCKSIZE;
        __int64 qwFileOffset = dwFileSize - dwRemainSize;
        PBYTE pSrc = (PBYTE)MapViewOfFile(hSrcFileMapping, FILE_MAP_READ, (DWORD)(qwFileOffset >> 32), (DWORD)(qwFileOffset & 0xFFFFFFFF), dwBlock);
        PBYTE pSrcBak = pSrc;
        for (int i = 0; i < dwBlock; i++)
        {
            //這樣的處理方式有一個很大的缺點
            //當整個文件的最後一行不是空一行的話，整個數據會少一行。
            //但是一般默認情況下整個數據的最後一行是有一個換行的
            if (*pSrc == '\n')
            {
                //整行讀完了====================================================
                if (FieldIndex != 0)//先處理一次字段。
                {
                    strLine[FieldIndex] = '\0';//在末尾處加上符號。
                    arrXYZ_RGB[FieldCount++] = atof(strLine);
                    FieldIndex = 0;
                }
                if (AlreadySetFiledCount == false)//如果沒有設置數據寬度
                {
                    //有用的字段大於3個吧
                    if (FieldCount == XYZ_FC || FieldCount == XYZI_FC || FieldCount == XYZRGB_FC || FieldCount == XYZIRGB_FC)
                    {
                        m_FieldCount = FieldCount;//數據寬度
                        usefulFiledCount = FieldCount;
                    }
                    else
                    {
                        m_FieldCount = XYZ_FC;//數據寬度默認爲3
                        usefulFiledCount = XYZ_FC;//如果沒有可用的數據類型，就默認爲3列
                    }
                    AlreadySetFiledCount = true;//已經設置了列寬
                }

                CloudVertex myVerPoint;
                if (usefulFiledCount == XYZ_FC)
                {
                    myVerPoint.pos.setX(arrXYZ_RGB[0]);
                    myVerPoint.pos.setY(arrXYZ_RGB[1]);
                    myVerPoint.pos.setZ(arrXYZ_RGB[2]);
                    myVerPoint.col.setX(1.0);
                    myVerPoint.col.setY(1.0);
                    myVerPoint.col.setZ(1.0);
                }
                if (usefulFiledCount == XYZI_FC)
                {
                    myVerPoint.pos.setX(arrXYZ_RGB[0]);
                    myVerPoint.pos.setY(arrXYZ_RGB[1]);
                    myVerPoint.pos.setZ(arrXYZ_RGB[2]);
                    myVerPoint.col.setX(1.0);
                    myVerPoint.col.setY(1.0);
                    myVerPoint.col.setZ(1.0);
                }
                if (usefulFiledCount == XYZRGB_FC)
                {
                    myVerPoint.pos.setX(arrXYZ_RGB[0]);
                    myVerPoint.pos.setY(arrXYZ_RGB[1]);
                    myVerPoint.pos.setZ(arrXYZ_RGB[2]);
                    myVerPoint.col.setX(arrXYZ_RGB[3]/255.0);
                    myVerPoint.col.setY(arrXYZ_RGB[4]/255.0);
                    myVerPoint.col.setZ(arrXYZ_RGB[5]/255.0);
                }
                if (usefulFiledCount == XYZIRGB_FC)
                {
                    myVerPoint.pos.setX(arrXYZ_RGB[0]);
                    myVerPoint.pos.setY(arrXYZ_RGB[1]);
                    myVerPoint.pos.setZ(arrXYZ_RGB[2]);
                    myVerPoint.col.setX(arrXYZ_RGB[4]/255.0);
                    myVerPoint.col.setY(arrXYZ_RGB[5]/255.0);
                    myVerPoint.col.setZ(arrXYZ_RGB[6]/255.0);
                }
                PointVec.push_back(myVerPoint);
                totalRows++;
                FieldCount = 0;//字段位置清零
                memset(strLine, 0, sizeof(strLine));//數字字符數組清空
            }
            else if ((*pSrc >= '0' && *pSrc <= '9') || *pSrc == '.' || *pSrc == '-' || *pSrc == 'e' || *pSrc == '+')
            {
                //空格 或Tab
                strLine[FieldIndex++] = *pSrc;
            }
            else
            {
                //此時爲行內分割===關鍵是連續幾次無用字符==============================
                if (FieldIndex != 0)
                {
                    //一個字段處理完畢
                    strLine[FieldIndex] = '\0';
                    arrXYZ_RGB[FieldCount++] = atof(strLine);
                    FieldIndex = 0;
                }
            }
            pSrc++;
        }
        UnmapViewOfFile(pSrcBak);
        dwRemainSize -= dwBlock;
    }
    CloseHandle(hSrcFileMapping);
    CloseHandle(hSrcFile);

    m_PointNum = PointVec.size();
    m_PointCloud = new CloudVertex[m_PointNum];
    for (size_t i = 0; i != PointVec.size(); i++)
    {
        *(m_PointCloud + i) = PointVec[i];
    }
    vector<CloudVertex>().swap(PointVec);//清空
    time = ((double)cv::getTickCount() - time) / cv::getTickFrequency();
    std::cout << "time cost:" << time << endl;
    std::cout << "total row:" << totalRows << endl;
    return totalRows;
}

    由於不同文件點雲的列數不確定，我們需要自適應進行判斷所以纔會有這麼多判斷。

    接下來是QTextStream的讀取方式：

//The Reading Speed of the QT Stream
double QTTextStreamTest(QString FilePath)
{
    double start=0,end=0;
    start = getTickCount();
    //other useful variables
    unsigned linesRead = 0;
    unsigned pointsRead = 0;
    char separator = ' ';
    QFile file(FilePath);
    if (!file.open(QFile::ReadOnly))
        return -1; //we clear already initialized data
    QTextStream stream(&file);
    QString currentLine = stream.readLine();
    QVector<QVector3D> CloudVec;
    while (!currentLine.isNull())
    {
        ++linesRead;
        if (currentLine.startsWith("//"))
        {
            currentLine = stream.readLine();
            continue;
        }
        if (currentLine.size() == 0)
        {
            currentLine = stream.readLine();
            continue;
        }
        //we split current line
        QStringList parts = currentLine.split(separator,QString::SkipEmptyParts);
        int nParts = parts.size();//for simple test we dont have other part
        QVector3D P;
        P.setX(parts[0].toDouble());
        P.setY(parts[1].toDouble());
        P.setZ(parts[2].toDouble());
        CloudVec.push_back(P);
        ++pointsRead;

        //read next line
        currentLine = stream.readLine();
    }
    file.close();
    end = getTickCount();
    qDebug()<<"Read Cloud Size:"<<CloudVec.size();
    double timeCost =(end - start)/getTickFrequency();
    return timeCost;//Return time cost
}

    最後是文件流讀取方式：

struct CloudStruct
{
    double x=0,y=0,z=0,r=0,g=0,b=0;
};
double CFileStreamTest(QString FilePath)
{
    std::string stdFilePath = FilePath.toStdString();
    ifstream CloudReader(stdFilePath);
    if(!CloudReader)
    {
        return -1;
    }
    double start=0,end=0;
    start = getTickCount();

    QVector<CloudStruct> CloudVec;
    CloudStruct Temp;
    while(CloudReader>>Temp.x)
    {
        CloudReader>>Temp.y;
        CloudReader>>Temp.z;
        CloudReader>>Temp.r;
        CloudReader>>Temp.g;
        CloudReader>>Temp.b;
        CloudVec.push_back(Temp);
    }
    CloudReader.close();
    end = getTickCount();
    qDebug()<<"Read Cloud Size:"<<CloudVec.size();
    double timeCost =(end - start)/getTickFrequency();
    return timeCost;
}

具體調用：

void qscarlet_opencvMaster::on_actionQTReadingSpeedTest_triggered()
{
    ui->tabWidget->setCurrentIndex(4);//change to Index 4
    QFileDialog *CloudfileDialog = new QFileDialog(this);
    CloudfileDialog->setFileMode(QFileDialog::ExistingFiles);//可以多選
    //fileDialog->setFileMode(QFileDialog::Directory);//變成了選擇文件夾的對話框
    CloudfileDialog->setWindowTitle(tr("Please Select a Cloud"));
    CloudfileDialog->setNameFilter(tr("Cloud Files (*.txt *.ptx *.pts *.ply *.asc *.xyz)"));
    CloudfileDialog->setDirectory(".");
    CloudfileDialog->setViewMode(QFileDialog::Detail);//List
    if(CloudfileDialog->exec() == QDialog::Accepted)
    {
        ui->ViewTabWidget->setCurrentIndex(2);//切換到3DView
        for(int i=0;i<CloudfileDialog->selectedFiles().size();i++)
        {
            QString path =CloudfileDialog->selectedFiles()[i];
            int64 start=0,end=0;
            start = getTickCount();
            m_GLWidget_ES2->AddCloudFromPath(path);
            end = getTickCount();
            cout << "C++ File Map Time Cost: " << (end - start)/getTickFrequency()<<"s"<< endl;
            SetCurrentFile(path);

            double ReadingTime = QTTextStreamTest(path);
            cout << "QT Text Stream Time Cost: " << ReadingTime<<"s"<< endl;

            ReadingTime = CFileStreamTest(path);
            cout << "C++ file Stream Time Cost: " << ReadingTime<<"s"<< endl;
        }
    }
    else
    {
        //QMessageBox::information(NULL, tr("Path"), tr("You didn't select any files."));
    }


}

最終的結果對比：

C++ File Map Time Cost: 14.5303s

Read Cloud Size: 10545264

QT Text Stream Time Cost: 16.3132s

Read Cloud Size: 10545264

C++ file Stream Time Cost: 179.773s

    可見速度最快的還是文件映射，其次是QTextStream，最慢的是C++文件流。

    然而在編程難度上，QTextStream要遠遠小於文件映射，而且基於QT的子類可以加入QProgress等進度條，這一點文件映射是做不到的。

    就到此爲止了。