C++詞法掃描程序-編譯原理實驗一

程序實驗功能：打開一個C++源文件，進行詞法分析，輸出代碼中的Token+Token類型，或者對C++源文件進行壓縮，去掉不必要空行、空 格、縮進、註釋等，生成一個壓縮版的C++代碼文件，再次對這個代碼文件進行詞法分析，也是能正確掃描出其中的Token

界面如下：

 

 

實現的類如下：詞法分析功能的類函數爲：Scan()，可以返回：“詞法Token+Token類型”，或者返回當前掃描出的Token

version 0.2

update：增加對源代碼的壓縮功能，以及訂正一個特殊符號的識別

public class Scanner { /// <summary> /// 保存token的動態數組 /// </summary> public ArrayList Tokens = new ArrayList(); //保存掃描到的所有token的數組 /// <summary> /// 掃描字符下標 /// </summary> private int index; /// <summary> /// 待掃描的字符串的長度 /// </summary> private int Length; /// <summary> /// 待掃描的字符串 /// </summary> private string ContentString; /// <summary> /// DFA狀態圖 /// </summary> private enum DFAState { Start, Identifier, Number, Operator, Strings, Comment, Special, Done }; /// <summary> /// 對讀取到的一個字符進行類型識別：數字、字符、特殊符號（3種） /// </summary> private enum CharType { digit, letter, other }; /// <summary> /// 保留字數組 /// </summary> private string[] ReservedWords = new string[]{"asm","auto","bool","break","case","catch","char","class", "const","const_cast","continue","default","delete","do","double","dynamic_cast","else","enum", "explicit","export","extern","false","float","for","friend","goto","if","inline","int","long","main", "mutable","namespace","new","operator","private","protected","public","register","reinterpret_cast", "return","short","signed","sizeof","static","static_cast","string","struct","switch","template", "this","throw","true","try","typedef","typeid","typename","union","unsigned","using","virtual","void", "volatile","wchar_t","while"}; /// <summary> /// 單個目運算符數組 /// </summary> private char[] SingleOp = new char[] { '.', ':', '+', '-', '*', '/', '%', '<', '>', '=', '!', '&', '|' }; /// <summary> /// 雙目運算符（如果一個字符屬於單目運算符，則再進行雙目的查找） /// </summary> private string[] DoubleOp = new string[] { "->", "++", "--", "<<", ">>", "<=", ">=", "==", "!=", "&&", "||", "+=", "-=","*=", "/=", "%=", "&=", "^=", "|=" }; /// <summary> /// 特殊字符 /// </summary> private char[] SpecialChar = new char[] { '#', ',', ';', '(', ')', '[', ']', '{', '}','~' }; /// <summary> /// 保留字的list表，用以BinarySearch()查找關鍵字 /// </summary> private List<string> rwList = new List<string>(); /// <summary> /// 雙目運算符的list表，用以BinarySearch()查找雙目運算符 /// </summary> private List<string> doblopList = new List<string>(); /// <summary> /// 當前token的類型，壓縮功能增加的變量 /// </summary> private DFAState currFlag = DFAState.Start; /// <summary> /// 前一個tokne的類型，壓縮功能增加的變量 /// </summary> private DFAState preFlag = DFAState.Start; /// <summary> /// 構造函數 /// </summary> /// <param name="str">進行詞法分析的源文件內容</param> public Scanner(string str) { index = -1; ContentString = str; Length = str.Length; foreach (string s in ReservedWords) //初始化rwList { rwList.Add(s); } rwList.Sort(); //對關鍵字列表進行快速排序 foreach (string s in DoubleOp) //初始化doblopList { doblopList.Add(s); } doblopList.Sort(); //對雙目運算符列表進行快速排序 } /// <summary> /// 對一個掃描到的字符進行類型的判斷 /// </summary> /// <param name="ch">字符</param> /// <returns>字符的類型</returns> private CharType TellaChar(char ch) { int t = (int)ch; if (t >= 48 && t <= 57) //數字 return CharType.digit; if (t >= 65 && t <= 90) //字母 return CharType.letter; if (t >= 97 && t <= 122) //字母 return CharType.letter; else return CharType.other; //可能運算符或特殊符號，或者空白或換行符 } /// <summary> /// 對一個字符進行匹配查找，判斷是否單目運算符 /// </summary> /// <param name="ch">字符</param> /// <returns>如果匹配，返回true</returns> private Boolean isSingleOp(char ch) { foreach (char c in SingleOp) if (c == ch) return true; return false; } /// <summary> /// 對一個長度爲2的字符串進行匹配，判斷是否雙目運算符 /// </summary> /// <param name="str">長度爲2的字符串</param> /// <returns>如果匹配，返回true</returns> private Boolean isDoubleOp(string str) { int result = doblopList.BinarySearch(str); if (result >= 0) return true; else return false; } /// <summary> /// 判斷一個字符是否爲特殊字符 /// </summary> /// <param name="ch">字符</param> /// <returns>如果時候特殊符號，返回true</returns> private Boolean isSpecialChar(char ch) { foreach (char c in SpecialChar) if (ch == c) return true; return false; } /// <summary> /// 取下一個字符 /// </summary> /// <returns>字符的ASCII值</returns> private char GetNextChar() { char ch; try { index++; ch = ContentString[index]; } catch (IndexOutOfRangeException ex) { throw ex; } return ch; } /// <summary> /// 保留字查找函數 /// </summary> /// <param name="str">標識符</param> /// <returns>如果找到匹配，說明該標識符是保留字，返回true</returns> private Boolean reservedWordLookup(string str) { int result = rwList.BinarySearch(str); if (result >= 0) return true; else return false; } /// <summary> /// 詞法掃描函數 version 0.2 /// </summary> /// <returns>掃描出的當前Token已經Token類型</returns> public string Scan() { string tkstString = null; //CurrentToken+CurrentState char ch; //保存讀取的一個字符 CharType chartype; //字符類型 string CurrentToken = null; //輸出的token DFAState CurrentState = DFAState.Start; //DFA圖狀態 while (index <= Length && CurrentState != DFAState.Done) { ch = GetNextChar(); //取下一個字符 chartype = TellaChar(ch); switch (CurrentState) { //開始狀態的處理 case DFAState.Start: if (ch == '/t' || ch == '/r' || ch == '/n' || ch == ' ') //回車換行空白 { CurrentState = DFAState.Start; } else if (ch == '_' || chartype == CharType.letter) { CurrentToken += ch; CurrentState = DFAState.Identifier; //標識符 } else if (chartype == CharType.digit) { CurrentToken += ch; CurrentState = DFAState.Number; //無符號數字 } else if (ch == '+' || ch == '-') { CurrentToken += ch; ch = GetNextChar(); //再取一個字符進行測試 chartype = TellaChar(ch); if (chartype == CharType.digit) { char t = ContentString[index - 2]; //對前前個字符進行判斷，比如a=b+3;取字符'b' CharType ct = TellaChar(t); if (ct == CharType.letter || ct == CharType.digit || t == ')' || t == ']') { index--; CurrentState = DFAState.Done; tkstString = CurrentToken + "/t運算符"; currFlag = DFAState.Operator; } else { CurrentToken += ch; CurrentState = DFAState.Number; //有符號數字，比如+5、-4 } } else { index--; //回退 CurrentState = DFAState.Operator; //運算符 } } else if (ch == '/') { CurrentToken += ch; ch = GetNextChar(); if (ch == '/' || ch == '*') { CurrentToken += ch; CurrentState = DFAState.Comment; //單行或多行註釋 } else { index--; CurrentState = DFAState.Operator; //運算符 } } else if (ch == '/'' || ch == '/"') // ',''字符或字符串的開頭字符 { CurrentToken += ch; CurrentState = DFAState.Strings; } else if (isSingleOp(ch)) //除了+、-、/的其他運算符 { CurrentToken += ch; CurrentState = DFAState.Operator; } else if (isSpecialChar(ch)) //特殊符號 { CurrentToken += ch; CurrentState = DFAState.Done; tkstString = CurrentToken + "/t特殊符號"; currFlag = DFAState.Special; } else { CurrentToken += ch; tkstString = CurrentToken + "/t/t程序未處理的bug！"; CurrentState = DFAState.Done; } break; //標識符狀態的處理 case DFAState.Identifier: if (ch == '_' || chartype == CharType.letter || chartype == CharType.digit) { CurrentToken += ch; CurrentState = DFAState.Identifier; } else { index--; //回退一個字符 CurrentState = DFAState.Done; //讀取完一個標識符 currFlag = DFAState.Identifier; //判斷是否是關鍵字 if (reservedWordLookup(CurrentToken)) { tkstString = CurrentToken + "/t關鍵字"; } else { tkstString = CurrentToken + "/t標識符"; } } break; //數字的處理 case DFAState.Number: if (ch == 'E') { CurrentToken += ch; ch = GetNextChar(); if (ch == '+' || ch == '-') { CurrentToken += ch; } else { index--; } CurrentState = DFAState.Number; } else if (chartype == CharType.digit || ch == '.') { CurrentToken += ch; CurrentState = DFAState.Number; } else { index--; //回退一個字符 currFlag = DFAState.Number; CurrentState = DFAState.Done; tkstString = CurrentToken + "/t數字"; } break; //操作符的處理，單目或雙目 case DFAState.Operator: string opt = CurrentToken + ch; if (isDoubleOp(opt)) //對單目運算符進行判斷，是否爲雙目運算符 { CurrentToken += ch; //雙目運算符 } else { index--; } CurrentState = DFAState.Done; tkstString = CurrentToken + "/t運算符"; currFlag = DFAState.Operator; break; //字符或字符串的處理 case DFAState.Strings: ; CurrentToken += ch; if (ch == '//') //如果字符串中含有轉義字符，則下一個字符也是包含在串中 { ch = GetNextChar(); CurrentToken += ch; CurrentState = DFAState.Strings; continue; //跳出這個case，回到while循環 } if (CurrentToken.StartsWith("/'") && ch == '/'') { tkstString = CurrentToken + "/t串"; CurrentState = DFAState.Done; currFlag = DFAState.Strings; } else if (CurrentToken.StartsWith("/"") && ch == '/"') { tkstString = CurrentToken + "/t串"; CurrentState = DFAState.Done; currFlag = DFAState.Strings; } else { CurrentState = DFAState.Strings; } break; //註釋行 case DFAState.Comment: if (CurrentToken.StartsWith("/*") == true)//單行或多行註釋 { CurrentToken += ch; if (ch == '*') { ch = GetNextChar(); CurrentToken += ch; CurrentState = DFAState.Comment; if (ch == '/') { CurrentState = DFAState.Done; tkstString = CurrentToken + "/t註釋"; currFlag = DFAState.Comment; } } else { CurrentState = DFAState.Comment; } } if (CurrentToken.StartsWith("//") == true) //單行註釋 { if (ch == '/n') //換行符表示一行註釋的結束 { CurrentState = DFAState.Done; tkstString = CurrentToken + "/t註釋"; currFlag = DFAState.Comment; } else { CurrentToken += ch; CurrentState = DFAState.Comment; } } break; //不應該執行這裏 default: CurrentToken = "ERROR SCAN HERE"; tkstString = CurrentToken + "/t出錯了"; CurrentState = DFAState.Done; break; }//switch(state) }//while(state!=State.Done) if (CurrentToken != null && currFlag != DFAState.Comment) { if (currFlag != DFAState.Special && currFlag == preFlag) { Tokens.Add(" "); } preFlag = currFlag; Tokens.Add(CurrentToken); } return tkstString; //return CurrentToken; }//scan() /// <summary> /// 判斷掃描是否結束 /// </summary> /// <returns></returns> public Boolean isFinishScan() { if (index <= Length) return true; else return false; } }

 

可執行文件(Lexical_Scanner.rar)下載：http://cid-780607117452312e.office.live.com/self.aspx/.Public/Lexical%20Scanner.rar

 

 

實驗二鏈接：http://blog.csdn.net/popozhu/archive/2010/06/28/5700229.aspx