ICTCLAS分詞系統研究(九)--對最終結果做優化調整

在研究(八)中,我們得到了最終的分詞結果了,好興奮呀。不過,還有臨門一腳不能忘了,對一些特殊情況做處理。主要是對疊詞(相鄰的兩個字或詞一樣)及個別詞性進行合併處理。

比如,以“一片片的白雲很好看”,他的最終分詞結果是:

 經過優化後的分詞結果:

序號 分詞結果
0 一/m 片/q 片/q 的/uj 白雲/n 很/d 好看/a

很顯然,“一片片”應該爲一個整體,沒有拆分的必要,看源代碼的調整過程:

//Adjust the result with some rules
bool CResult::Adjust(PWORD_RESULT pItem,PWORD_RESULT pItemRet)
{
    
int i=0,j=0;
    unsigned 
int nLen;
    
char sSurName[10],sSurName2[10],sGivenName[10];
    
bool bProcessed=false;//Have been processed
    while(pItem[i].sWord[0]!=0)
    
{
        nLen
=strlen(pItem[i].sWord);
        bProcessed
=false;
        
        
//Rule1: adjust person name
        if(pItem[i].nHandle==28274&&ChineseNameSplit(pItem[i].sWord,sSurName,sSurName2,sGivenName,m_uPerson.m_dict)&&strcmp(pItem[i].sWord,"葉利欽")!=0)//'nr'
        {//Divide name into surname and given name
            
            
if(sSurName[0])
            
{
                strcpy(pItemRet[j].sWord,sSurName);
                pItemRet[j
++].nHandle=28274;
            }

            
if(sSurName2[0])
            
{
                strcpy(pItemRet[j].sWord,sSurName2);
                pItemRet[j
++].nHandle=28274;
            }

            
if(sGivenName[0])
            
{
                strcpy(pItemRet[j].sWord,sGivenName);
                pItemRet[j
++].nHandle=28274;
            }

            bProcessed
=true;
        }

        
//Rule2 for overlap words ABB 一段段、一片片
        else if(pItem[i].nHandle==27904&&strlen(pItem[i+1].sWord)==2&&strcmp(pItem[i+1].sWord,pItem[i+2].sWord)==0)
        
{//(pItem[i+1].nHandle/256=='q'||pItem[i+1].nHandle/256=='a')&&
            strcpy(pItemRet[j].sWord,pItem[i].sWord);
            strcat(pItemRet[j].sWord,pItem[i
+1].sWord);
            strcat(pItemRet[j].sWord,pItem[i
+2].sWord);
            pItemRet[j].nHandle
=27904;
            j
+=1;
            i
+=2;
            bProcessed
=true;
        }

        
//Rule3 for overlap words AA
        else if(nLen==2&&strcmp(pItem[i].sWord,pItem[i+1].sWord)==0)
        
{
            strcpy(pItemRet[j].sWord,pItem[i].sWord);
            strcat(pItemRet[j].sWord,pItem[i
+1].sWord);
             
//24832=='a'*256
            pItemRet[j].nHandle=24832;//a
            if(pItem[i].nHandle/256=='v'||pItem[i+1].nHandle/256=='v')//30208='v'8256
            {
                pItemRet[j].nHandle
=30208;
            }

            
if(pItem[i].nHandle/256=='n'||pItem[i+1].nHandle/256=='n')//30208='v'8256
            {
                pItemRet[j].nHandle
='n'*256;
            }
            
            i
+=1;
            
if(strlen(pItem[i+1].sWord)==2)
            
{//AAB:洗/洗/臉、矇矇亮
                if((pItemRet[j].nHandle==30208&&pItem[i+1].nHandle/256=='n')||
                   (pItemRet[j].nHandle
==24832&&pItem[i+1].nHandle/256=='a')
                   )
                
{
                    strcat(pItemRet[j].sWord,pItem[i
+1].sWord);
                    i
+=1;
                }

            }

            j
+=1;
            bProcessed
=true;
        }


        
//Rule 4: AAB 洗/洗澡
        else if(nLen==2&&strncmp(pItem[i].sWord,pItem[i+1].sWord,2)==0&&strlen(pItem[i+1].sWord)==4&&(pItem[i].nHandle/256=='v'||pItem[i].nHandle==24832))//v,a
        {
            strcpy(pItemRet[j].sWord,pItem[i].sWord);
            strcat(pItemRet[j].sWord,pItem[i
+1].sWord);
             
//24832=='a'*256
            pItemRet[j].nHandle=24832;//'a'
            if(pItem[i].nHandle/256=='v'||pItem[i+1].nHandle/256=='v')//30208='v'8256
            {
                pItemRet[j].nHandle
=30208;
            }


            i
+=1;
            j
+=1;
            bProcessed
=true;
        }

        
else if(pItem[i].nHandle/256=='u'&&pItem[i].nHandle%256)//uj,ud,uv,uz,ul,ug->u
            pItem[i].nHandle='u'*256;
        
else if(nLen==2&&strncmp(pItem[i].sWord,pItem[i+1].sWord,2)==0&&strlen(pItem[i+1].sWord)==4&&strncmp(pItem[i+1].sWord+2,pItem[i+2].sWord,2)==0)
        
{//AABB 樸樸素素 枝枝葉葉
                strcpy(pItemRet[j].sWord,pItem[i].sWord);
                strcat(pItemRet[j].sWord,pItem[i
+1].sWord);
                strcat(pItemRet[j].sWord,pItem[i
+2].sWord);
                pItemRet[j].nHandle
=pItem[i+1].nHandle;
                i
+=2;
                j
+=1;
                bProcessed
=true;
        }

        
else if(pItem[i].nHandle==28275)//PostFix
        {
            
if(m_uPlace.m_dict.IsExist(pItem[i+1].sWord,4))
            
{
                strcpy(pItemRet[j].sWord,pItem[i].sWord);
                strcat(pItemRet[j].sWord,pItem[i
+1].sWord);
                pItemRet[j].nHandle
=28275;
                i
+=1;
                j
+=1;
                bProcessed
=true;
            }

            
else if(strlen(pItem[i+1].sWord)==2&&CC_Find("",pItem[i+1].sWord))
            
{
                strcpy(pItemRet[j].sWord,pItem[i].sWord);
                strcat(pItemRet[j].sWord,pItem[i
+1].sWord);
                pItemRet[j].nHandle
=28276;
                i
+=1;
                j
+=1;
                bProcessed
=true;
            }

            
else if(strlen(pItem[i+1].sWord)==2&&CC_Find("語文字杯",pItem[i+1].sWord))
            
{
                strcpy(pItemRet[j].sWord,pItem[i].sWord);
                strcat(pItemRet[j].sWord,pItem[i
+1].sWord);
                pItemRet[j].nHandle
=28282;
                i
+=1;
                j
+=1;
                bProcessed
=true;
            }

            
else if(strlen(pItem[i+1].sWord)==2&&CC_Find("",pItem[i+1].sWord))
            
{
                strcpy(pItemRet[j].sWord,pItem[i].sWord);
                strcat(pItemRet[j].sWord,pItem[i
+1].sWord);
                pItemRet[j].nHandle
=28160;
                i
+=1;
                j
+=1;
                bProcessed
=true;
            }

        }

        
else if(pItem[i].nHandle==30208||pItem[i].nHandle==28160)//v
        {
            
if(strlen(pItem[i+1].sWord)==2&&CC_Find("",pItem[i+1].sWord))
            
{
                strcpy(pItemRet[j].sWord,pItem[i].sWord);
                strcat(pItemRet[j].sWord,pItem[i
+1].sWord);
                pItemRet[j].nHandle
=28160;
                i
+=1;
                j
+=1;
                bProcessed
=true;
            }

        }

        
else if(pItem[i].nHandle==28280)
        
{//www/nx ./w sina/nx; EIM/nx  -601/m 
            strcpy(pItemRet[j].sWord,pItem[i].sWord);
            pItemRet[j].nHandle
=28280;
            
while(pItem[i+1].nHandle==28280||strstr("..",pItem[i+1].sWord)||(pItem[i+1].nHandle==27904&&IsAllNum((unsigned char *)pItem[i+1].sWord)))
            
{
                strcat(pItemRet[j].sWord,pItem[i
+1].sWord);
                i
+=1;
            }

            j
+=1;
            bProcessed
=true;
        }


        
if(!bProcessed)
        
{//If not processed,that's mean: not need to adjust;
         
//just copy to the final result
            strcpy(pItemRet[j].sWord,pItem[i].sWord);
            pItemRet[j
++].nHandle=pItem[i].nHandle;
        }

        i
++;
    }

    pItemRet[j].sWord[
0]=0;//Set ending
    return true;
}

 

調整合並後的最終分詞結果爲:

一片片/m 的/u 白雲/n 很/d 好看/a

至此,我們終於完成了整個分詞的流程,得到了正確的切分結果^_^

發佈了65 篇原創文章 · 獲贊 5 · 訪問量 53萬+
發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章