HttpRunner3源碼閱讀：5. 參數/函數調用及其值處理

parser

上一篇讀的loader.py，裏面提到的就是文件路徑,文件轉用例模型、套件模型、加載方法字典，變量寫入環境，這篇parser.py主要內容是在解析用例當中引用變量、自定義方法
變量和方法表達式和實際項目衝突的時候就需要改這個文件了

可用資料

https://docs.python.org/zh-cn/3/library/re.html?highlight=re#module-re

導包

import ast  # 內置庫： 抽象語法樹
import builtins # 內建對象 該模塊提供對Python的所有“內置”標識符的直接訪問
import re   # 內置庫 正則表達式
import os   # 內置庫 系統
from typing import Any, Set, Text, Callable, List, Dict, Union

from loguru import logger
from sentry_sdk import capture_exception

from httprunner import loader, utils, exceptions  
from httprunner.models import VariablesMapping, FunctionsMapping

源碼附註釋

# re.compile 返回一個正則表達式對象， re.I 忽略大小寫匹配：忽略大小寫匹配
# 匹配http url 的正則表達式對象
absolute_http_url_regexp = re.compile(r"^https?://", re.I)

# use $$ to escape $ notation
dolloar_regex_compile = re.compile(r"\$\$")
# variable notation, e.g. ${var} or $var
# 引用變量 查找 正則表達式
variable_regex_compile = re.compile(r"\$\{(\w+)\}|\$(\w+)")
# function notation, e.g. ${func1($var_1, $var_3)}  方法
# (\w+) 匹配的函數名  ([\$\w\.\-/\s=,]*) 匹配的參數
function_regex_compile = re.compile(r"\$\{(\w+)\(([\$\w\.\-/\s=,]*)\)\}")


def parse_string_value(str_value: Text) -> Any:
    """ parse string to number if possible
    e.g. "123" => 123
         "12.2" => 12.3
         "abc" => "abc"
         "$var" => "$var"
    """
    try:
        # 字符串轉數字,"'123'" 轉的則是 123 => str 類型
        return ast.literal_eval(str_value)
    except ValueError:
        return str_value
    except SyntaxError:
        # e.g. $var, ${func}
        return str_value


def build_url(base_url, path):
    """拼接請求地址
    base_url => http://www.baidu.com/
    path => /search/name=httprunner
    return => http://www.baidu.com/search/name=httprunner
    """
    
    """ prepend url with base_url unless it's already an absolute URL """
    if absolute_http_url_regexp.match(path):    # 如果是http開頭則認爲他是完整的url
        return path
    elif base_url:  # 進行url 拼接
        return "{}/{}".format(base_url.rstrip("/"), path.lstrip("/"))
    else:
        raise exceptions.ParamsError("base url missed!")

# 正則表達式查變量
def regex_findall_variables(raw_string: Text) -> List[Text]:
    """ extract all variable names from content, which is in format $variable

    Args:
        raw_string (str): string content

    Returns:
        list: variables list extracted from string content

    Examples:
        >>> regex_findall_variables("$variable")
        ["variable"]

        >>> regex_findall_variables("/blog/$postid")
        ["postid"]

        >>> regex_findall_variables("/$var1/$var2")
        ["var1", "var2"]

        >>> regex_findall_variables("abc")
        []

    """
    try:
        # 返回 $ 在 raw_string 從下標0 開始第一次出現的下標
        match_start_position = raw_string.index("$", 0)
    except ValueError:  # 找不到直接退出這個方法
        return []

    vars_list = []
    while match_start_position < len(raw_string):
        # Notice: notation priority
        # $$ > $var

        # search $$ 
        dollar_match = dolloar_regex_compile.match(raw_string, match_start_position)
        if dollar_match:
            # 找到了就返回re.match 中的結束下標， 賦值 然後 結束此次循環
            match_start_position = dollar_match.end()
            continue

        # search variable like ${var} or $var 找 ${} 、 $x
        var_match = variable_regex_compile.match(raw_string, match_start_position)
        if var_match:
            # age${name}$info => var_name => ${name}
            # var_match.end() 10(下標9)
            var_name = var_match.group(1) or var_match.group(2)
            vars_list.append(var_name)
            match_start_position = var_match.end()
            continue

        curr_position = match_start_position
        try:
            # find next $ location # 上述幾個if 之後還有的話把從 match_start_position + 1 的下標開始找$出現的座標繼續
            match_start_position = raw_string.index("$", curr_position + 1)
        except ValueError:
            # break while loop
            break
    # 最終得到一個參數列表["${name}","$age",...]
    return vars_list

# 正則查找方法 ${func()}
def regex_findall_functions(content: Text) -> List[Text]:
    """ extract all functions from string content, which are in format ${fun()}

    Args:
        content (str): string content

    Returns:
        list: functions list extracted from string content

    Examples:
        >>> regex_findall_functions("${func(5)}")
        ["func(5)"]

        >>> regex_findall_functions("${func(a=1, b=2)}")
        ["func(a=1, b=2)"]

        >>> regex_findall_functions("/api/1000?_t=${get_timestamp()}")
        ["get_timestamp()"]

        >>> regex_findall_functions("/api/${add(1, 2)}")
        ["add(1, 2)"]

        >>> regex_findall_functions("/api/${add(1, 2)}?_t=${get_timestamp()}")
        ["add(1, 2)", "get_timestamp()"]
        
            result = function_regex_compile.findall("age${name(1,2)}$info${name1()}${aoligei($name)}}")
print(result)  # [('name', '1,2'), ('name1', ''), ('aoligei', '$name')]
    """
    try:
        # re.findall 返回一個不重複的 pattern 的匹配列表
        return function_regex_compile.findall(content)
    except TypeError as ex:
        capture_exception(ex)
        return []

# 遞歸提取變量
def extract_variables(content: Any) -> Set:
    """ extract all variables in content recursively.
    """
    if isinstance(content, (list, set, tuple)):
        variables = set()
        for item in content:
            # 兩個集合 並集
            variables = variables | extract_variables(item)
        return variables

    elif isinstance(content, dict):
        variables = set()
        for key, value in content.items():
            variables = variables | extract_variables(value)
        return variables

    elif isinstance(content, str):
        return set(regex_findall_variables(content))

    return set()

# 方法參數解析
def parse_function_params(params: Text) -> Dict:
    """ parse function params to args and kwargs.

    Args:
        params (str): function param in string

    Returns:
        dict: function meta dict

            {
                "args": [],
                "kwargs": {}
            }

    Examples:
        >>> parse_function_params("")
        {'args': [], 'kwargs': {}}

        >>> parse_function_params("5")
        {'args': [5], 'kwargs': {}}

        >>> parse_function_params("1, 2")
        {'args': [1, 2], 'kwargs': {}}

        >>> parse_function_params("a=1, b=2")
        {'args': [], 'kwargs': {'a': 1, 'b': 2}}

        >>> parse_function_params("1, 2, a=3, b=4")
        {'args': [1, 2], 'kwargs': {'a':3, 'b':4}}

    """
    function_meta = {"args": [], "kwargs": {}}

    params_str = params.strip()  # 去除首尾空格
    if params_str == "":
        return function_meta

    args_list = params_str.split(",") # ,拆分參數列表
    for arg in args_list:
        arg = arg.strip()
        if "=" in arg:  # 關鍵字入參
            key, value = arg.split("=")
            function_meta["kwargs"][key.strip()] = parse_string_value(value.strip())
        else:
            function_meta["args"].append(parse_string_value(arg))

    return function_meta

# 從變量池獲取參數
def get_mapping_variable(
    variable_name: Text, variables_mapping: VariablesMapping
) -> Any:
    """ get variable from variables_mapping.

    Args:
        variable_name (str): variable name
        variables_mapping (dict): variables mapping

    Returns:
        mapping variable value.

    Raises:
        exceptions.VariableNotFound: variable is not found.

    """
    # TODO: get variable from debugtalk module and environ
    try:
        return variables_mapping[variable_name]
    except KeyError:
        raise exceptions.VariableNotFound(
            f"{variable_name} not found in {variables_mapping}"
        )

# 從方法池裏面取方法對象
def get_mapping_function(
    function_name: Text, functions_mapping: FunctionsMapping
) -> Callable:
    """ get function from functions_mapping,
        if not found, then try to check if builtin function.

    Args:
        function_name (str): function name
        functions_mapping (dict): functions mapping

    Returns:
        mapping function object.

    Raises:
        exceptions.FunctionNotFound: function is neither defined in debugtalk.py nor builtin.

    """
    if function_name in functions_mapping:
        return functions_mapping[function_name]
     # 參數化
    elif function_name in ["parameterize", "P"]:
        return loader.load_csv_file
     # 環境文件       
    elif function_name in ["environ", "ENV"]:
        return utils.get_os_environ
    # 上傳文件
    elif function_name in ["multipart_encoder", "multipart_content_type"]:
        # extension for upload test
        from httprunner.ext import uploader

        return getattr(uploader, function_name)

    try:
        # 預置方法
        # check if HttpRunner builtin functions
        built_in_functions = loader.load_builtin_functions()
        return built_in_functions[function_name]
    except KeyError:
        pass

    try:
        # check if Python builtin functions
        return getattr(builtins, function_name)
    except AttributeError:
        pass

    raise exceptions.FunctionNotFound(f"{function_name} is not found.")

# 解析字符串 
def parse_string(
    raw_string: Text,
    variables_mapping: VariablesMapping,
    functions_mapping: FunctionsMapping,
) -> Any:
    """ parse string content with variables and functions mapping.

    Args:
        raw_string: raw string content to be parsed.
        variables_mapping: variables mapping.
        functions_mapping: functions mapping.

    Returns:
        str: parsed string content.

    Examples:
        >>> raw_string = "abc${add_one($num)}def"
        >>> variables_mapping = {"num": 3}
        >>> functions_mapping = {"add_one": lambda x: x + 1}
        >>> parse_string(raw_string, variables_mapping, functions_mapping)
            "abc4def"

    """
    try:
        # 1.查找$開頭座標
        match_start_position = raw_string.index("$", 0)
        # 2. 截取到不需要處理的字符串內容
        parsed_string = raw_string[0:match_start_position]
    except ValueError:
        # 上面找不到時出現異常 結束 原封不動的返回字符串所有內容
        parsed_string = raw_string
        return parsed_string
    # 3. 循環查找
    while match_start_position < len(raw_string):

        # Notice: notation priority
        # $$ > ${func($a, $b)} > $var

        # search $$
        # 4. 從 上面第一次找到$的座標點開始查找$$，找不到返回None， 找到就返回一個對象
        dollar_match = dolloar_regex_compile.match(raw_string, match_start_position)
        if dollar_match:
            # 5. 返回 $$ 最後$的 座標 + 1
            match_start_position = dollar_match.end()
            # 6. 不必變化的內容 追加 $
            parsed_string += "$"
            continue
        """4.5.6例子
        str2 = "sdfsadf$${123},$$1123123${func(1,2)}demo"
i = str2.index("$", 0)
pars = str2[0:i]
print(i, pars)  # 7  sdfsadf

# $$
result = dolloar_regex_compile.match("sdfsadf$${123},$$1123123$", i)
print(result.end())  # 9
pars += "$"
print(str2,pars) # sdfsadf$${123},$$1123123${func(1,2)}demo sdfsadf$

        """

        # search function like ${func($a, $b)} 都是以$ 開頭 匹配 函數引用表達式
        func_match = function_regex_compile.match(raw_string, match_start_position)
        """
        result = function_regex_compile.match("${func1($var_1, $var_3)}", 0)
print(result.groups())  # ('func1', '$var_1, $var_3')
        """

        if func_match:
            # 7. 獲取函數名
            func_name = func_match.group(1)
            # 8. 從函數字典中拿到 函數對象
            func = get_mapping_function(func_name, functions_mapping)
            # 9. 得到函數的參數 如 '$var_1, $var_3'
            func_params_str = func_match.group(2)
            # 10. 解析成位置入參/ 關鍵字入參
            function_meta = parse_function_params(func_params_str)
            args = function_meta["args"] # 順序入參 傳遞
            kwargs = function_meta["kwargs"] # 關鍵字參數傳遞
            # 11.函數參數 變量轉換處理  $name =>  zy7y  
            parsed_args = parse_data(args, variables_mapping, functions_mapping)  # 順序位置參數 
            parsed_kwargs = parse_data(kwargs, variables_mapping, functions_mapping) # 關鍵字參數

            try:
                # 12.調用函數(對象)，回想apiAutoTest 還是用的內置函數exec 執行 字符函數表達式(欠佳呀)
                func_eval_value = func(*parsed_args, **parsed_kwargs)
            except Exception as ex:
                logger.error(
                    f"call function error:\n"
                    f"func_name: {func_name}\n"
                    f"args: {parsed_args}\n"
                    f"kwargs: {parsed_kwargs}\n"
                    f"{type(ex).__name__}: {ex}"
                )
                raise
            # 13. 將函數執行結果 和表達式進行替換
            func_raw_str = "${" + func_name + f"({func_params_str})" + "}"
            if func_raw_str == raw_string:
                # raw_string is a function, e.g. "${add_one(3)}", return its eval value directly
                return func_eval_value

            # raw_string contains one or many functions, e.g. "abc${add_one(3)}def"
            # 14. 包含多個函數表達式 結果轉成字符串拼接在不變的位置
            parsed_string += str(func_eval_value)
            # 15. 拿到} 下標 + 1, 下次查找開始位置
            match_start_position = func_match.end()
            continue

        # search variable like ${var} or $var
        # 16. 變量查找替換
        var_match = variable_regex_compile.match(raw_string, match_start_position)
        if var_match:
            var_name = var_match.group(1) or var_match.group(2)
            # 17. 從變量池拿到對應值
            var_value = get_mapping_variable(var_name, variables_mapping)

            if f"${var_name}" == raw_string or "${" + var_name + "}" == raw_string:
                # raw_string is a variable, $var or ${var}, return its value directly
                return var_value

            # raw_string contains one or many variables, e.g. "abc${var}def"
            parsed_string += str(var_value)
            match_start_position = var_match.end()
            continue

        curr_position = match_start_position
        try:
            # find next $ location
            match_start_position = raw_string.index("$", curr_position + 1)
            # 尾巴不需要處理部分的內容
            remain_string = raw_string[curr_position:match_start_position]
        except ValueError:
            # 從截取最後$ + 1 開始的位置到最後 內容
            remain_string = raw_string[curr_position:]
            # break while loop
            match_start_position = len(raw_string)
        # 處理完成 拼接字符串
        parsed_string += remain_string

    return parsed_string

# 處理變量池 變量映射# 變量池換成具體數據
def parse_data(
    raw_data: Any,
    variables_mapping: VariablesMapping = None,
    functions_mapping: FunctionsMapping = None,
) -> Any:
    """ parse raw data with evaluated variables mapping.
        Notice: variables_mapping should not contain any variable or function.
    """
    if isinstance(raw_data, str):
        # content in string format may contains variables and functions
        variables_mapping = variables_mapping or {}
        functions_mapping = functions_mapping or {}
        # only strip whitespaces and tabs, \n\r is left because they maybe used in changeset
        raw_data = raw_data.strip(" \t")
        # 調用上面的處理字符串
        return parse_string(raw_data, variables_mapping, functions_mapping)

    elif isinstance(raw_data, (list, set, tuple)):
        # 列表推導式
        return [
            parse_data(item, variables_mapping, functions_mapping) for item in raw_data
        ]

    elif isinstance(raw_data, dict):
        parsed_data = {}
        for key, value in raw_data.items():
            parsed_key = parse_data(key, variables_mapping, functions_mapping)
            parsed_value = parse_data(value, variables_mapping, functions_mapping)
            parsed_data[parsed_key] = parsed_value

        return parsed_data

    else:
        # other types, e.g. None, int, float, bool
        return raw_data

# 解析變量池
def parse_variables_mapping(
    variables_mapping: VariablesMapping, functions_mapping: FunctionsMapping = None
) -> VariablesMapping:

    parsed_variables: VariablesMapping = {}

    while len(parsed_variables) != len(variables_mapping):
        for var_name in variables_mapping:

            if var_name in parsed_variables:
                continue
            # 從池子拿到對應value
            var_value = variables_mapping[var_name]
            # 變量名列表
            variables = extract_variables(var_value)  # ["var1", "var2"]

            # check if reference variable itself
            if var_name in variables:
                # e.g.
                # variables_mapping = {"token": "abc$token"}
                # variables_mapping = {"key": ["$key", 2]}
                raise exceptions.VariableNotFound(var_name)

            # check if reference variable not in variables_mapping
            not_defined_variables = [
                v_name for v_name in variables if v_name not in variables_mapping
            ]
            if not_defined_variables:
                # e.g. {"varA": "123$varB", "varB": "456$varC"}
                # e.g. {"varC": "${sum_two($a, $b)}"}
                raise exceptions.VariableNotFound(not_defined_variables)

            try:
                # 返回實際變量對應的值
                parsed_value = parse_data(
                    var_value, parsed_variables, functions_mapping
                )
            except exceptions.VariableNotFound:
                continue
            # 返回解析變量池
            parsed_variables[var_name] = parsed_value

    return parsed_variables


# 笛卡爾積生成 + 解析參數
def parse_parameters(parameters: Dict,) -> List[Dict]:
    """ parse parameters and generate cartesian product.

    Args:
        parameters (Dict) parameters: parameter name and value mapping
            parameter value may be in three types:
                (1) data list, e.g. ["iOS/10.1", "iOS/10.2", "iOS/10.3"]
                (2) call built-in parameterize function, "${parameterize(account.csv)}"
                (3) call custom function in debugtalk.py, "${gen_app_version()}"

    Returns:
        list: cartesian product list

    Examples:
        >>> parameters = {
            "user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"],
            "username-password": "${parameterize(account.csv)}",
            "app_version": "${gen_app_version()}",
        }
        >>> parse_parameters(parameters)

    """
    parsed_parameters_list: List[List[Dict]] = []

    # load project_meta functions
    project_meta = loader.load_project_meta(os.getcwd())
    functions_mapping = project_meta.functions

    for parameter_name, parameter_content in parameters.items():
        parameter_name_list = parameter_name.split("-")

        if isinstance(parameter_content, List):
            # (1) data list
            # e.g. {"app_version": ["2.8.5", "2.8.6"]}
            #       => [{"app_version": "2.8.5", "app_version": "2.8.6"}]
            # e.g. {"username-password": [["user1", "111111"], ["test2", "222222"]}
            #       => [{"username": "user1", "password": "111111"}, {"username": "user2", "password": "222222"}]
            parameter_content_list: List[Dict] = []
            for parameter_item in parameter_content:
                if not isinstance(parameter_item, (list, tuple)):
                    # "2.8.5" => ["2.8.5"]
                    parameter_item = [parameter_item]

                # ["app_version"], ["2.8.5"] => {"app_version": "2.8.5"}
                # ["username", "password"], ["user1", "111111"] => {"username": "user1", "password": "111111"}
                parameter_content_dict = dict(zip(parameter_name_list, parameter_item))
                parameter_content_list.append(parameter_content_dict)

        elif isinstance(parameter_content, Text):
            # (2) & (3)
            parsed_parameter_content: List = parse_data(
                parameter_content, {}, functions_mapping
            )
            if not isinstance(parsed_parameter_content, List):
                raise exceptions.ParamsError(
                    f"parameters content should be in List type, got {parsed_parameter_content} for {parameter_content}"
                )

            parameter_content_list: List[Dict] = []
            for parameter_item in parsed_parameter_content:
                if isinstance(parameter_item, Dict):
                    # get subset by parameter name
                    # {"app_version": "${gen_app_version()}"}
                    # gen_app_version() => [{'app_version': '2.8.5'}, {'app_version': '2.8.6'}]
                    # {"username-password": "${get_account()}"}
                    # get_account() => [
                    #       {"username": "user1", "password": "111111"},
                    #       {"username": "user2", "password": "222222"}
                    # ]
                    parameter_dict: Dict = {
                        key: parameter_item[key] for key in parameter_name_list
                    }
                elif isinstance(parameter_item, (List, tuple)):
                    if len(parameter_name_list) == len(parameter_item):
                        # {"username-password": "${get_account()}"}
                        # get_account() => [("user1", "111111"), ("user2", "222222")]
                        parameter_dict = dict(zip(parameter_name_list, parameter_item))
                    else:
                        raise exceptions.ParamsError(
                            f"parameter names length are not equal to value length.\n"
                            f"parameter names: {parameter_name_list}\n"
                            f"parameter values: {parameter_item}"
                        )
                elif len(parameter_name_list) == 1:
                    # {"user_agent": "${get_user_agent()}"}
                    # get_user_agent() => ["iOS/10.1", "iOS/10.2"]
                    # parameter_dict will get: {"user_agent": "iOS/10.1", "user_agent": "iOS/10.2"}
                    parameter_dict = {parameter_name_list[0]: parameter_item}
                else:
                    raise exceptions.ParamsError(
                        f"Invalid parameter names and values:\n"
                        f"parameter names: {parameter_name_list}\n"
                        f"parameter values: {parameter_item}"
                    )

                parameter_content_list.append(parameter_dict)

        else:
            raise exceptions.ParamsError(
                f"parameter content should be List or Text(variables or functions call), got {parameter_content}"
            )

        parsed_parameters_list.append(parameter_content_list)

    return utils.gen_cartesian_product(*parsed_parameters_list)

最後

這一節很多地方沒理解到爲什麼要怎麼處理， 但確實被調用自定義函數給亮到了

• parse_variables_mapping
• parse_parameters
  留下的問題 後面整體運行遇到了在回來看吧...