python的ORM運用“extra”、“annotate”

多對一，一多關係的三張表：

class SystemUser(CommonInfo):

    class Meta:
        verbose_name = "系統用戶"

    bk_username = models.CharField("用戶名稱", max_length=50, unique=True)
    qq = models.CharField("qq賬號", max_length=20, default="")
    bk_role = models.SmallIntegerField("用戶角色")
    phone = models.CharField("電話號碼", max_length=20)
    wx_userid = models.CharField("微信ID", max_length=50)
    email = models.CharField("郵箱地址", max_length=100)
    chname = models.CharField("中文名稱", max_length=50)
    group = models.ManyToManyField(SystemGroup, verbose_name="人員分組", related_name="users")

    def to_dict(self):
        return_dict = {f.name: getattr(self, f.name) for f in self._meta.fields}
        user_notifys = self.notifies.all()
        return_dict["sms_notify_enable"] = user_notifys.filter(type=UserNotify.SMS, is_enable=True).exists()
        return_dict["email_notify_enable"] = user_notifys.filter(type=UserNotify.EMAIL, is_enable=True).exists()
        return_dict["wechat_notify_enable"] = user_notifys.filter(type=UserNotify.WECHAT, is_enable=True).exists()
        return_dict["api_notify_enable"] = user_notifys.filter(type=UserNotify.API, is_enable=True).exists()
        return_dict["groups"] = list(self.group.values())
        return return_dict

    def get_ins_summary(self):
        return self._meta.verbose_name + '[用戶名稱: {0}, 中文名稱: {1}]'.format(self.bk_username, self.chname)

class UserAlarm(models.Model):

    class Meta:
        verbose_name = "用戶告警事件"
        unique_together = ["system_user", "alarm"]

    DISPATCH = "dispatch"
    CLAIM = "claim"
    CLOSE = "close"
    CREATE_REASON_CHOICES = (
        (DISPATCH, "分派"),
        (CLAIM, "認領"),
        (CLOSE, "關閉")
    )

    system_user = models.ForeignKey("system_mgmt.SystemUser", verbose_name="系統用戶")
    alarm = models.ForeignKey(AlarmActive, verbose_name="告警事件")
    created_time = models.DateTimeField("創建時間", auto_now_add=True, db_index=True)
    response_time = models.DateTimeField("響應時間", null=True, db_index=True)
    close_time = models.DateTimeField("關閉時間", null=True, db_index=True)
    create_reason = models.CharField("創建原因", max_length=15, choices=CREATE_REASON_CHOICES, db_index=True)

    def to_dict(self):
        return_dict = {f.name: getattr(self, f.name) for f in self._meta.fields if
                       f.name not in ["system_user", "alarm"]}
        return_dict["create_reason_display"] = self.get_create_reason_display()
        return_dict.update(self.alarm.to_dict())
        return return_dict

class AlarmActive(models.Model):

    class Meta:
        verbose_name = "告警事件定義"

    FIRING = "firing"
    RESOLVED = "resolved"
    ACTION_CHOICES = (
        (FIRING, "產生告警"),
        (RESOLVED, "消除告警")
    )

    REMAIN = "remain"
    WARNING = "warning"
    FATAL = "fatal"
    LEVEL_CHOICES = (
        (REMAIN, "提醒"),
        (WARNING, "警告"),
        (FATAL, "致命")
    )

    ABNORMAL = "abnormal"
    CLOSED = "closed"
    SHIELDED = "shielded"
    PENDING_EXECUTE = "pending_execute"
    EXECUTING = "executing"
    DISPATCHED = "dispatched"
    RESTORED = "restored"
    STATUS_CHOICES = (
        (ABNORMAL, "未分派"),
        (CLOSED, "已關閉"),
        (SHIELDED, "已屏蔽"),
        (PENDING_EXECUTE, "待審批"),
        (EXECUTING, "處理中"),
        (DISPATCHED, "待響應"),
        (RESTORED, "已恢復")
    )

    id = models.CharField("ID", max_length=80, default=gen_id_by_uuid, primary_key=True)
    source_type = models.CharField('告警源', max_length=64, db_index=True)
    alarm_type = models.CharField('告警指標', max_length=64, db_index=True)
    alarm_name = models.CharField('告警名稱', max_length=255, db_index=True, default="")
    event_id = models.CharField("事件ID", max_length=60, db_index=True)
    alarm_time = models.DateTimeField("告警時間", auto_now_add=True, db_index=True)
    close_time = models.DateTimeField("關閉時間", null=True, db_index=True)
    alarm_content = models.TextField("原始告警類容", default="")
    action = models.CharField("告警動作", max_length=15, choices=ACTION_CHOICES, default=FIRING, db_index=True)
    level = models.CharField("告警級別", max_length=15, choices=LEVEL_CHOICES, db_index=True)
    status = models.CharField("告警處理狀態", max_length=20, choices=STATUS_CHOICES, default=ABNORMAL, db_index=True)

    # 藍鯨平臺相關字段
    ip = models.CharField("告警源IP", max_length=30, default="", db_index=True)
    bk_biz_id = models.SmallIntegerField("業務ID", default=0, db_index=True)
    bk_set_id = models.SmallIntegerField("集羣ID", default=0, db_index=True)
    bk_module_id = models.SmallIntegerField("模塊ID", default=0, db_index=True)
    bk_cloud_id = models.SmallIntegerField("雲區域ID", default=0, db_index=True)

    # 保留字段, 未知字段
    tag_info = models.CharField("標籤", default="", max_length=255, db_index=True)
    meta_info = models.CharField("元信息", default="", max_length=255, db_index=True)

    def to_dict(self):
        return_dict = {f.name: getattr(self, f.name) for f in self._meta.fields}
        return_dict["action_display"] = self.get_action_display()
        return_dict["level_display"] = self.get_level_display()
        return_dict["status_display"] = self.get_status_display()
        return return_dict

demo：求折線圖。

def _get_month_day_list(begin_date, end_date):
    month_day_list = []
    begin_date = datetime.datetime.strptime(begin_date, '%Y-%m-%d')
    end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d')
    i = datetime.timedelta(days=0)
    while i <= (end_date-begin_date):
        month_day_list.append((begin_date+i).strftime('%Y-%m-%d'))
        i += datetime.timedelta(days=1)

    return month_day_list

def _get_join_line(all_active_alarms, begin_date, end_date):

    month_day_list = _get_month_day_list(begin_date, end_date)
    # <type 'list'>: ['2019-10-08', '2019-10-09', '2019-10-10', '2019-10-11', '2019-10-12', '2019-10-13', '2019-10-14', '2019-10-15', '2019-10-16', '2019-10-17']

    month_day_join_num_dict = {i: 0 for i in month_day_list}
    # {'2019-10-09': 0, '2019-10-08': 0, '2019-10-12': 0, '2019-10-13': 0, '2019-10-10': 0, '2019-10-11': 0, '2019-10-16': 0, '2019-10-17': 0, '2019-10-14': 0, '2019-10-15': 0}

    y_data = []
    source_type_join_num_query_list = \
        all_active_alarms.extra(select={'year-month-day': "DATE_FORMAT(alarm_time, '%%Y-%%c-%%d')"}).\
        values("year-month-day", "source_type").annotate(join_num=Count("alarm_time"))
# [{u'year-month-day': u'2019-10-09', 'join_num': 5, u'source_type': u'REST_API_PULL'}, {u'year-month-day': u'2019-10-08', 'join_num': 6, u'source_type': u'REST_API_PULL'}, {u'year-month-day': u'2019-10-11', 'join_num': 3, u'source_type': u'REST_API_PULL'}]

    source_type_join_num_dict = defaultdict(dict)
    for i in source_type_join_num_query_list:
        source_type_join_num_dict[i["source_type"]][i["year-month-day"]] = i["join_num"]

    for source_type, num_dict in source_type_join_num_dict.items():
        default_dict = copy.copy(month_day_join_num_dict)
        default_dict.update(num_dict)
        y_data.append({"name": source_type, "type": "line", "data": map(lambda x: default_dict[x], month_day_list)})

    return month_day_list, y_data

談談對上述代碼的理解：

功能：通過傳入active_alarms實例和begin時間和end時間。求一段時間內按照某種類型某種時間分類後統計改表數據的條數。

month_day_list求得是一個時間range，類型是list。

month_day_join_num_dict是爲了統計該時間範圍內一定時間內統計的數量，key爲時間（str)，value初始化爲0（int），類型爲dict。

extra(select={'year-month-day': "DATE_FORMAT(alarm_time, '%%Y-%%c-%%d')"})選擇一個時間鍵做一個格式轉變，賦給一個臨時變量year-month-day。

values("year-month-day", "source_type") 選擇出需要的字段

annotate(join_num=Count("alarm_time"))以時間統計，但是分類好像是以上面選擇出來的關鍵字作爲分類的憑據。

分類出來後的數據是：[{'year-month-day': u'2019-10-09', 'join_num': 5, 'source_type': u'REST_API_PULL'}, {'year-month-day': u'2019-10-08', 'join_num': 6, 'source_type': u'REST_API_PULL'}, {'year-month-day': u'2019-10-11', 'join_num': 3, 'source_type': u'REST_API_PULL'}]

然後source_type_join_num_dict = defaultdict(dict)  初始化一個空字典。

for i in source_type_join_num_query_list:
    source_type_join_num_dict[i["source_type"]][i["year-month-day"]] = i["join_num"]

將之前的結果進行一個分類處理，第一個分類是類型，第二個分類是時間。

for source_type, num_dict in source_type_join_num_dict.items():
    default_dict = copy.copy(month_day_join_num_dict)
    default_dict.update(num_dict)
    y_data.append({"name": source_type, "type": "line", "data": map(lambda x: default_dict[x], month_day_list)})

格式處理成想要的數據格式，這裏比較厲害的操作就是複製上面的month_day_join_num_dict作爲初始化的數據，然後更新相同的時間key：default_dict.update(num_dict)。

還有這裏"data": map(lambda x: default_dict[x], month_day_list) 字典是無序的，利用有序的列表取字典裏面的value，然後重新創建一個只有統計值的列表。