Linux下如何使用awk解析json數據

近期在做一個項目，調用api後返回了一個嵌套較多層的json格式的數據，常規解析json的方式也已經很難或做不到提取出需要的信息點，恰好對awk比較熟悉，考慮到awk本身也是一種用於數據處理的工具，於是就有了以下用awk解析json的嘗試。下面以face++ 的detect api返回值，json格式：

{
  "faces": [
    {
      "face_token": "826536dba3973d71d219a15e18c9597b",
      "face_rectangle": {
        "height": 113,
        "left": 198,
        "top": 125,
        "width": 113
      },
      "attributes": {
        "age": {
          "value": 26
        },
        "gender": {
          "value": "Female"
        }
      },
      "landmark": {
        "mouth_upper_lip_bottom": {
          "x": 257,
          "y": 203
        },
        "right_eye_upper_right_quarter": {
          "x": 284,
          "y": 142
        },
        "left_eyebrow_upper_right_quarter": {
          "x": 235,
          "y": 127
        },
        "mouth_upper_lip_right_contour3": {
          "x": 267,
          "y": 202
        },
        "left_eye_top": {
          "x": 228,
          "y": 143
        },
        "nose_contour_right3": {
          "x": 265,
          "y": 185
        },
        "contour_right6": {
          "x": 298,
          "y": 207
        },
        "mouth_lower_lip_bottom": {
          "x": 257,
          "y": 211
        },
        "nose_contour_right2": {
          "x": 268,
          "y": 172
        },
        "nose_contour_right1": {
          "x": 264,
          "y": 149
        },
        "nose_contour_left3": {
          "x": 250,
          "y": 185
        },
        "nose_contour_left2": {
          "x": 246,
          "y": 172
        },
        "nose_contour_left1": {
          "x": 249,
          "y": 150
        },
        "right_eyebrow_upper_right_quarter": {
          "x": 286,
          "y": 128
        },
        "left_eyebrow_upper_middle": {
          "x": 226,
          "y": 126
        },
        "right_eye_upper_left_quarter": {
          "x": 273,
          "y": 143
        },
        "right_eyebrow_right_corner": {
          "x": 292,
          "y": 133
        },
        "contour_right8": {
          "x": 283,
          "y": 227
        },
        "right_eyebrow_upper_left_quarter": {
          "x": 271,
          "y": 127
        },
        "contour_right2": {
          "x": 309,
          "y": 159
        },
        "contour_right3": {
          "x": 309,
          "y": 172
        },
        "contour_right1": {
          "x": 309,
          "y": 147
        },
        "left_eye_lower_left_quarter": {
          "x": 222,
          "y": 149
        },
        "contour_left1": {
          "x": 201,
          "y": 150
        },
        "contour_left2": {
          "x": 201,
          "y": 162
        },
        "contour_left3": {
          "x": 203,
          "y": 175
        },
        "contour_left4": {
          "x": 205,
          "y": 187
        },
        "contour_left5": {
          "x": 209,
          "y": 199
        },
        "contour_left6": {
          "x": 215,
          "y": 210
        },
        "contour_left7": {
          "x": 224,
          "y": 220
        },
        "right_eye_bottom": {
          "x": 280,
          "y": 149
        },
        "right_eye_lower_right_quarter": {
          "x": 284,
          "y": 148
        },
        "right_eyebrow_left_corner": {
          "x": 266,
          "y": 132
        },
        "right_eyebrow_upper_middle": {
          "x": 279,
          "y": 126
        },
        "mouth_lower_lip_top": {
          "x": 257,
          "y": 205
        },
        "right_eyebrow_lower_left_quarter": {
          "x": 272,
          "y": 132
        },
        "left_eyebrow_lower_left_quarter": {
          "x": 219,
          "y": 133
        },
        "mouth_upper_lip_left_contour3": {
          "x": 247,
          "y": 203
        },
        "left_eyebrow_lower_middle": {
          "x": 226,
          "y": 132
        },
        "left_eye_upper_left_quarter": {
          "x": 222,
          "y": 144
        },
        "mouth_upper_lip_left_contour1": {
          "x": 252,
          "y": 198
        },
        "mouth_upper_lip_top": {
          "x": 257,
          "y": 199
        },
        "mouth_upper_lip_left_contour2": {
          "x": 244,
          "y": 200
        },
        "right_eye_pupil": {
          "x": 279,
          "y": 145
        },
        "mouth_lower_lip_right_contour1": {
          "x": 267,
          "y": 204
        },
        "mouth_lower_lip_left_contour2": {
          "x": 242,
          "y": 207
        },
        "mouth_lower_lip_right_contour3": {
          "x": 265,
          "y": 210
        },
        "mouth_lower_lip_right_contour2": {
          "x": 272,
          "y": 206
        },
        "contour_chin": {
          "x": 259,
          "y": 237
        },
        "contour_left9": {
          "x": 245,
          "y": 235
        },
        "left_eye_lower_right_quarter": {
          "x": 233,
          "y": 150
        },
        "mouth_left_corner": {
          "x": 236,
          "y": 202
        },
        "contour_right4": {
          "x": 307,
          "y": 184
        },
        "contour_right7": {
          "x": 291,
          "y": 217
        },
        "left_eyebrow_left_corner": {
          "x": 211,
          "y": 135
        },
        "nose_right": {
          "x": 272,
          "y": 181
        },
        "nose_tip": {
          "x": 257,
          "y": 177
        },
        "contour_right5": {
          "x": 303,
          "y": 196
        },
        "nose_contour_lower_middle": {
          "x": 258,
          "y": 187
        },
        "right_eye_top": {
          "x": 278,
          "y": 141
        },
        "mouth_lower_lip_left_contour3": {
          "x": 249,
          "y": 210
        },
        "right_eye_right_corner": {
          "x": 288,
          "y": 146
        },
        "mouth_upper_lip_right_contour1": {
          "x": 261,
          "y": 198
        },
        "mouth_upper_lip_right_contour2": {
          "x": 270,
          "y": 199
        },
        "right_eyebrow_lower_right_quarter": {
          "x": 285,
          "y": 132
        },
        "left_eye_left_corner": {
          "x": 218,
          "y": 148
        },
        "mouth_right_corner": {
          "x": 278,
          "y": 201
        },
        "right_eye_lower_left_quarter": {
          "x": 275,
          "y": 149
        },
        "left_eyebrow_right_corner": {
          "x": 242,
          "y": 132
        },
        "left_eyebrow_lower_right_quarter": {
          "x": 234,
          "y": 132
        },
        "right_eye_center": {
          "x": 279,
          "y": 146
        },
        "left_eye_pupil": {
          "x": 228,
          "y": 146
        },
        "nose_left": {
          "x": 243,
          "y": 182
        },
        "mouth_lower_lip_left_contour1": {
          "x": 247,
          "y": 204
        },
        "left_eye_upper_right_quarter": {
          "x": 233,
          "y": 145
        },
        "right_eyebrow_lower_middle": {
          "x": 279,
          "y": 132
        },
        "left_eye_center": {
          "x": 228,
          "y": 148
        },
        "contour_left8": {
          "x": 233,
          "y": 228
        },
        "contour_right9": {
          "x": 272,
          "y": 234
        },
        "right_eye_left_corner": {
          "x": 270,
          "y": 149
        },
        "left_eyebrow_upper_left_quarter": {
          "x": 218,
          "y": 128
        },
        "left_eye_bottom": {
          "x": 227,
          "y": 150
        },
        "left_eye_right_corner": {
          "x": 237,
          "y": 149
        }
      }
    }
  ],
  "time_used": 251,
  "request_id": "1492784536,22b21204-0e1e-4071-91ba-b29e5a68fe03",
  "image_id": "6kox5v0GjhI9f+MWEc3wIA=="
}

我們從調用api開始講起，以下是我對face++的一個調用，直接運用shell命令的curl發送的post請求：

curl -X POST "https://api-cn.faceplusplus.com/facepp/v3/detect" \
        -F "api_key=RF_13WYz9Cgz9KXtJxaqAIbIF7_jxuC-" \
        -F "api_secret=MeCM2hq-SSiWXgxMt-JcpOo-t5IAIdz4" \
        -F "image_file=@/home/gec/timg.jpeg" \
        -F "return_landmark=1" -F "return_attributes=gender,age"

爲了方便接下來awk對數據的解析，我們可以將調用api的返回值重定向到一個文件裏，將以上命令改寫成：

curl -X POST "https://api-cn.faceplusplus.com/facepp/v3/detect" \
        -F "api_key=RF_13WYz9Cgz9KXtJxaqAIbIF7_jxuC-" \
        -F "api_secret=MeCM2hq-SSiWXgxMt-JcpOo-t5IAIdz4" \
        -F "image_file=@/home/gec/timg.jpeg" \
        -F "return_landmark=1" -F "return_attributes=gender,age" >face

如此，當調用成功時，我們就可以在face中看到face++的detect api的正確返回值，查看文件用cat指令或者直接用文本編輯器打開都可以，如下：

cat face

{"image_id": "6kox5v0GjhI9f+MWEc3wIA==", "request_id": "1492784536,22b21204-0e1e-4071-91ba-b29e5a68fe03", "time_used": 251, "faces": [{"landmark": {"mouth_upper_lip_left_contour2": {"y": 200, "x": 244}, "mouth_upper_lip_top": {"y": 199, "x": 257}, "mouth_upper_lip_left_contour1": {"y": 198, "x": 252}, "left_eye_upper_left_quarter": {"y": 144, "x": 222}, "left_eyebrow_lower_middle": {"y": 132, "x": 226}, "mouth_upper_lip_left_contour3": {"y": 203, "x": 247}, "left_eyebrow_lower_left_quarter": {"y": 133, "x": 219}, "right_eyebrow_lower_left_quarter": {"y": 132, "x": 272}, "right_eye_pupil": {"y": 145, "x": 279}, "mouth_lower_lip_right_contour1": {"y": 204, "x": 267}, "mouth_lower_lip_left_contour2": {"y": 207, "x": 242}, "mouth_lower_lip_right_contour3": {"y": 210, "x": 265}, "mouth_lower_lip_right_contour2": {"y": 206, "x": 272}, "contour_chin": {"y": 237, "x": 259}, "contour_left9": {"y": 235, "x": 245}, "left_eye_lower_right_quarter": {"y": 150, "x": 233}, "mouth_lower_lip_top": {"y": 205, "x": 257}, "right_eyebrow_upper_middle": {"y": 126, "x": 279}, "right_eyebrow_left_corner": {"y": 132, "x": 266}, "right_eye_lower_right_quarter": {"y": 148, "x": 284}, "right_eye_bottom": {"y": 149, "x": 280}, "contour_left7": {"y": 220, "x": 224}, "contour_left6": {"y": 210, "x": 215}, "contour_left5": {"y": 199, "x": 209}, "contour_left4": {"y": 187, "x": 205}, "contour_left3": {"y": 175, "x": 203}, "contour_left2": {"y": 162, "x": 201}, "contour_left1": {"y": 150, "x": 201}, "left_eye_lower_left_quarter": {"y": 149, "x": 222}, "contour_right1": {"y": 147, "x": 309}, "contour_right3": {"y": 172, "x": 309}, "contour_right2": {"y": 159, "x": 309}, "mouth_left_corner": {"y": 202, "x": 236}, "contour_right4": {"y": 184, "x": 307}, "contour_right7": {"y": 217, "x": 291}, "left_eyebrow_left_corner": {"y": 135, "x": 211}, "nose_right": {"y": 181, "x": 272}, "nose_tip": {"y": 177, "x": 257}, "contour_right5": {"y": 196, "x": 303}, "nose_contour_lower_middle": {"y": 187, "x": 258}, "right_eye_top": {"y": 141, "x": 278}, "mouth_lower_lip_left_contour3": {"y": 210, "x": 249}, "right_eye_right_corner": {"y": 146, "x": 288}, "mouth_upper_lip_right_contour1": {"y": 198, "x": 261}, "mouth_upper_lip_right_contour2": {"y": 199, "x": 270}, "right_eyebrow_lower_right_quarter": {"y": 132, "x": 285}, "left_eye_left_corner": {"y": 148, "x": 218}, "mouth_right_corner": {"y": 201, "x": 278}, "right_eye_lower_left_quarter": {"y": 149, "x": 275}, "left_eyebrow_right_corner": {"y": 132, "x": 242}, "left_eyebrow_lower_right_quarter": {"y": 132, "x": 234}, "right_eye_center": {"y": 146, "x": 279}, "left_eye_pupil": {"y": 146, "x": 228}, "nose_left": {"y": 182, "x": 243}, "mouth_lower_lip_left_contour1": {"y": 204, "x": 247}, "left_eye_upper_right_quarter": {"y": 145, "x": 233}, "right_eyebrow_lower_middle": {"y": 132, "x": 279}, "left_eye_center": {"y": 148, "x": 228}, "contour_left8": {"y": 228, "x": 233}, "contour_right9": {"y": 234, "x": 272}, "right_eye_left_corner": {"y": 149, "x": 270}, "left_eyebrow_upper_left_quarter": {"y": 128, "x": 218}, "left_eye_bottom": {"y": 150, "x": 227}, "left_eye_right_corner": {"y": 149, "x": 237}, "right_eyebrow_upper_left_quarter": {"y": 127, "x": 271}, "contour_right8": {"y": 227, "x": 283}, "right_eyebrow_right_corner": {"y": 133, "x": 292}, "right_eye_upper_left_quarter": {"y": 143, "x": 273}, "left_eyebrow_upper_middle": {"y": 126, "x": 226}, "right_eyebrow_upper_right_quarter": {"y": 128, "x": 286}, "nose_contour_left1": {"y": 150, "x": 249}, "nose_contour_left2": {"y": 172, "x": 246}, "nose_contour_left3": {"y": 185, "x": 250}, "nose_contour_right1": {"y": 149, "x": 264}, "nose_contour_right2": {"y": 172, "x": 268}, "mouth_lower_lip_bottom": {"y": 211, "x": 257}, "contour_right6": {"y": 207, "x": 298}, "nose_contour_right3": {"y": 185, "x": 265}, "left_eye_top": {"y": 143, "x": 228}, "mouth_upper_lip_right_contour3": {"y": 202, "x": 267}, "left_eyebrow_upper_right_quarter": {"y": 127, "x": 235}, "right_eye_upper_right_quarter": {"y": 142, "x": 284}, "mouth_upper_lip_bottom": {"y": 203, "x": 257}}, "attributes": {"gender": {"value": "Female"}, "age": {"value": 26}}, "face_rectangle": {"width": 113, "top": 125, "left": 198, "height": 113}, "face_token": "826536dba3973d71d219a15e18c9597b"}]}

原本json結構是相當清晰的，每條數據都會按行分開，但是經實驗Linux下（也可能不是環境原因，這裏不是很清楚，還請朋友們不吝指正）返回的json是不分行的，所有數據都擠在一行上，這對解析來說無疑又加大了難度。

接下來嘗試從json當中提取需要的信息，以年齡age爲例。如果信息太多覺得不好找，可以用grep指令標記一下，這樣就可以很方便的找到關鍵詞的所在位置：

grep age face

博主最初的思路是：awk能不能通過匹配到age這個詞，然後輸出當前位置若干字符後的字符串26？博主按這個思路嘗試了很久，同時也發現網上有很多朋友問過類似的問題，但是都得不到答案，大概awk本身並沒有能直接實現這種需求的語法吧。

我最終想出來的方法是awk指定分隔符結合管道運用，指定 所需數據附近的符號 作爲分隔符將原json數據切塊，越切越小，切到最後保留的那塊數據就是我們想要的數據。

這個方法的優點在於無視了json格式的多層嵌套進行了直接切塊，完整指令如下：

awk -F '"' '{print $(NF-14)}' face|awk -F} '{print $1}'|awk -F:' ' '{print $2}'

將上面的指令剖析開來

第一步：

awk -F '"' '{print $(NF-14)}' face         #指定引號爲分隔符，打印倒數第14個字段

得到：

: 26}}, 

第二步，基於上面的結果再進行：

awk -F} '{print $1}'                        #指定｝爲分隔符，打印第1個字段

得到：

: 26

第三步，基於上面結果再進行：

awk -F:' ' '{print $2}'                     #指定：和空格 爲分隔符，打印第2個字段

得到結果：

26

至此，awk順利提取得到了json格式中的數據。由於json格式都是差不多的形式，指定分隔符加管道的方法同樣適用於其他api返回數據的解析，僅需修改字段數即可。

另外，linux下使用jq也是一個不錯的選擇：linux下使用jq解析json