python連接mysql獲取表信息（表名、字段數、字段空值率）

功能說明：

通過python連接mysql數據庫，1. 獲取當前庫下所有表的名稱 2. 獲取每個表的記錄行數 3. 查詢每張表下每個字段的空值率（採用增量查詢的方式，降低查詢大表時的性能消耗）。

輸出形式：

excel表格

代碼：

import math
import sys
import pymysql
import xlwt
import pandas as pd
# 數據庫信息
host = sys.argv[1]
user = sys.argv[2]
passwd = sys.argv[3]
db = sys.argv[4]
# excel的存放目錄
dir = sys.argv[5]
# 寫入EXCEL
def write_to_excel(res, sheet_name, excel_name):
    '''
    :param res: 要寫入excel的數據
    :param sheet_name: sheet頁名稱
    :param excel_name: excel名稱
    '''
    workbook = xlwt.Workbook()
    sheet = workbook.add_sheet(sheet_name, cell_overwrite_ok=True)
    row = 0
    for i in res:
        for j in range(0, len(i)):
            sheet.write(row, j, i[j])
        row += 1
    workbook.save(excel_name)
# 當前庫下所有表的名稱
def tables_name(db):
    res = mysql_info("select table_name from information_schema.tables where table_schema='%s'" % (db))
    print('當前庫下所有表的名稱')
    for i in res:
        print(i[0])
    return res
# 每個表的記錄行數
def count_rows(db):
    res = mysql_info("select table_name,table_rows from information_schema.tables where TABLE_SCHEMA = '%s'" % (db))
    print('每個表的記錄行數')
    print(res)
    return res
# 查詢每張表下每個字段的空值率並直接寫入excel
def null_rate(db, sheet_name, excel_name):
    '''
    :param db: 數據庫
    :param sheet_name: sheet頁名稱
    :param excel_name: excel名稱
    '''
    conn = pymysql.connect(host, user, passwd, db)
    workbook = xlwt.Workbook()
    sheet = workbook.add_sheet(sheet_name, cell_overwrite_ok=True)
    tables = mysql_info("SHOW TABLES")
    row = 0
    # 遍歷數據庫下所有的表
    for table in tables:
        # 獲取單表名稱
        table = str(table).replace("('", '').replace("',)", '')
        print('當前表名稱:', table)
        # 獲取主鍵名稱
        sql_for_search_key = "SELECT k.column_name FROM information_schema.table_constraints t JOIN information_schema.key_column_usage k USING (constraint_name,table_schema,table_name) WHERE t.constraint_type = 'PRIMARY KEY' AND t.table_schema = '%s' AND t.table_name = '%s' " % (
            db, table)
        pri_key = mysql_info(sql_for_search_key)[0][0]
        # excel寫入表名
        sheet.write(row, 0, table)
        sum_rows = mysql_info('select count(' + pri_key + ') from %s' % table)
        print(sum_rows)
        # 分頁查詢時每頁數目
        page_rows = 5000
        offset = 0
        # 創建一個空的df，用來存放字段及其對應空值數
        df = pd.DataFrame()
        # 這裏通過增量查取，降低性能開銷
        for page in range(1, math.ceil(sum_rows[0][0] / page_rows) + 1):
            # 查詢語句
            sql = 'select * from ' + table + ' order by ' + pri_key + ' limit ' + str(page_rows) + ' offset ' + str(
                offset)
            offset += page_rows
            # 將查詢數據轉換爲dataFrame，並追加到總的df中
            data = pd.read_sql(sql, con=conn)
            df = df.append(data)
        # 統計df每列爲空值的數量
        axis_data = df.isnull().sum(axis=0)
        # 將查詢結果寫入excel
        index_name = df.columns.values.tolist()
        axis_data = axis_data.values
        for i in range(0, len(index_name)):
            # 寫入字段名稱
            sheet.write(row, 1, index_name[i])
            # 寫入該字段空值率
            sheet.write(row, 2, format((axis_data[i] / int(sum_rows[0][0])), '.2%'))
            row += 1
    workbook.save(excel_name)
def mysql_info(sql):
    """
    :param sql: 執行的查詢sql語句
    :return: 返回查詢結果
    如果程中發生錯誤直接報錯退出
    """
    try:
        conn1 = pymysql.connect(host, user, passwd, db)
        cursor1 = conn1.cursor()
        cursor1.execute(sql)
        res = cursor1.fetchall()
        conn1.close()
        return res
    except Exception as e:
        print(e)
        print("!!!!!!!!!!!!!!請檢查數據庫連接信息!!!!!!!!!!!!!!")
        exit(-1)
if __name__ == "__main__":
    result1 = tables_name(db)
    write_to_excel(result1, 'tables_name', dir + '/%s庫中每個表的名字.xlsx' % db)
    result2 = count_rows(db)
    write_to_excel(result2, 'count_rows', dir + '/%s庫中每個表的記錄行數.xlsx' % db)
    null_rate(db, 'null_rate', dir + '/%s庫中每張表下每個字段的空值率.xlsx' % db)