quic協議棧中的數據包發送過程

 打算再從quic協議中挖點礦，我就需要了解quic中的數據包是怎麼打包，序列化的。早期的quic代碼較少，看起來比較直接，我看的是libquic。
 QuicPacketCreator::SerializePacket這個函數對queued_frames_中的數據進行了序列化。而一次的封包操作，數據包的長度爲max_plaintext_size_，具體大小不太清楚，總之，不超過kMaxPacketSize(1452)。

void QuicPacketCreator::SerializePacket(char* encrypted_buffer,
                                        size_t encrypted_buffer_len)
{
  size_t length = framer_->BuildDataPacket(header, queued_frames_,
                                           encrypted_buffer, packet_size_);
}                                        

 疑惑的是，這裏的queued_frames_中的所有幀的長度不超過max_plaintext_size_的大小的？發送的數據，要是超過了這個長度，多餘的數據該存放在哪裏呢？數據被緩存之後，什麼時候會被再次發送出去呢？ Just such curiosities motive me to read the fucking source code. Why the coders’ god Linus using the F* word? As such puzzles get solved after reading the source code, a pleasure is generated!
 從數據的發送開始，也就是ReliableQuicStream::WriteOrBufferData。

void ReliableQuicStream::WriteOrBufferData(
    StringPiece data,
    bool fin,
    QuicAckListenerInterface* ack_listener) {
      if (queued_data_.empty()) {
    struct iovec iov(MakeIovec(data));
    consumed_data = WritevData(&iov, 1, fin, ack_listener);
    DCHECK_LE(consumed_data.bytes_consumed, data.length());
  }

  // If there's unconsumed data or an unconsumed fin, queue it.
  //沒有寫完，則將剩餘的數據緩寸，因爲數據包的長度超出了kMaxPacketSize大小
  if (consumed_data.bytes_consumed < data.length() ||
      (fin && !consumed_data.fin_consumed)) {
    StringPiece remainder(data.substr(consumed_data.bytes_consumed));
    queued_data_bytes_ += remainder.size();
    queued_data_.emplace_back(remainder.as_string(), ack_listener);
  }
  }
  //當sent_packet_manager允許發送的時候，就是擁塞控制允許向外發送，緩存的數據被髮送出去。
  void ReliableQuicStream::OnCanWrite(){
   while (!queued_data_.empty()) {
    PendingData* pending_data = &queued_data_.front();
    QuicAckListenerInterface* ack_listener = pending_data->ack_listener.get();
    if (queued_data_.size() == 1 && fin_buffered_) {
      fin = true;
    }
    if (pending_data->offset > 0 &&
        pending_data->offset >= pending_data->data.size()) {
      // This should be impossible because offset tracks the amount of
      // pending_data written thus far.
      QUIC_BUG << "Pending offset is beyond available data. offset: "
               << pending_data->offset << " vs: " << pending_data->data.size();
      return;
    }
    size_t remaining_len = pending_data->data.size() - pending_data->offset;
    struct iovec iov = {
        const_cast<char*>(pending_data->data.data()) + pending_data->offset,
        remaining_len};
    QuicConsumedData consumed_data = WritevData(&iov, 1, fin, ack_listener);
    queued_data_bytes_ -= consumed_data.bytes_consumed;
    if (consumed_data.bytes_consumed == remaining_len &&
        fin == consumed_data.fin_consumed) {
      queued_data_.pop_front();
    } else {
      if (consumed_data.bytes_consumed > 0) {
        pending_data->offset += consumed_data.bytes_consumed;
      }
      break;
    }
  } 
  }
  QuicConsumedData ReliableQuicStream::WritevData(
    const struct iovec* iov,
    int iov_count,
    bool fin,
    QuicAckListenerInterface* ack_listener) {
      QuicConsumedData consumed_data =
      WritevDataInner(QuicIOVector(iov, iov_count, write_length),
                      stream_bytes_written_, fin, ack_listener);
   }
   QuicConsumedData ReliableQuicStream::WritevDataInner(
    QuicIOVector iov,
    QuicStreamOffset offset,
    bool fin,
    QuicAckListenerInterface* ack_notifier_delegate) {
  return session()->WritevData(this, id(), iov, offset, fin,
                               ack_notifier_delegate);
}
QuicConsumedData QuicSession::WritevData(
    ReliableQuicStream* stream,
    QuicStreamId id,
    QuicIOVector iov,
    QuicStreamOffset offset,
    bool fin,
    QuicAckListenerInterface* ack_notifier_delegate) {
      QuicConsumedData data =
      connection_->SendStreamData(id, iov, offset, fin, ack_notifier_delegate);
    }
    QuicConsumedData QuicConnection::SendStreamData(
    QuicStreamId id,
    QuicIOVector iov,
    QuicStreamOffset offset,
    bool fin,
    QuicAckListenerInterface* listener){
      return packet_generator_.ConsumeData(id, iov, offset, fin, listener);
    }

 上面顯示的兩個函數均有調用 consumed_data = WritevData(&iov, 1, fin, ack_listener)。能保證寫進queued_frames_中的所有幀長不會超過一個向網絡中發送一個UDP數據包的長度的操作就是 packet_generator_.ConsumeData。

QuicConsumedData QuicPacketGenerator::ConsumeData(
    QuicStreamId id,
    QuicIOVector iov,
    QuicStreamOffset offset,
    bool fin,
    QuicAckListenerInterface* listener){
    //當擁塞控制不允許寫的時候，或者數據包全部被Consume，纔會退出
  while (delegate_->ShouldGeneratePacket(
      HAS_RETRANSMITTABLE_DATA, has_handshake ? IS_HANDSHAKE : NOT_HANDSHAKE)) {
    QuicFrame frame;
    if (!packet_creator_.ConsumeData(id, iov, total_bytes_consumed,
                                     offset + total_bytes_consumed, fin,
                                     has_handshake, &frame)) {
      // The creator is always flushed if there's not enough room for a new
      // stream frame before ConsumeData, so ConsumeData should always succeed.
      //因爲下面調用了packet_creator_.Flush();，ConsumeData每次調用都會返回true
      QUIC_BUG << "Failed to ConsumeData, stream:" << id;
      return QuicConsumedData(0, false);
    }

    // A stream frame is created and added.
    size_t bytes_consumed = frame.stream_frame->data_length;
    if (listener != nullptr) {
      packet_creator_.AddAckListener(listener, bytes_consumed);
    }
    total_bytes_consumed += bytes_consumed;
    fin_consumed = fin && total_bytes_consumed == iov.total_length;
    DCHECK(total_bytes_consumed == iov.total_length ||
           (bytes_consumed > 0 && packet_creator_.HasPendingFrames()));

    if (!InBatchMode()) {
      packet_creator_.Flush();
    }

    if (total_bytes_consumed == iov.total_length) {
      // We're done writing the data. Exit the loop.
      // We don't make this a precondition because we could have 0 bytes of data
      // if we're simply writing a fin.
      break;
    }
    // TODO(ianswett): Move to having the creator flush itself when it's full.
    packet_creator_.Flush();
  }

  // Don't allow the handshake to be bundled with other retransmittable frames.
  if (has_handshake) {
    SendQueuedFrames(/*flush=*/true);
  }

  DCHECK(InBatchMode() || !packet_creator_.HasPendingFrames());
  return QuicConsumedData(total_bytes_consumed, fin_consumed);
   }
   bool QuicPacketCreator::ConsumeData(QuicStreamId id,
                                    QuicIOVector iov,
                                    size_t iov_offset,
                                    QuicStreamOffset offset,
                                    bool fin,
                                    bool needs_full_padding,
                                    QuicFrame* frame)
{
  CreateStreamFrame(id, iov, iov_offset, offset, fin, frame);
  if (!AddFrame(*frame, /*save_retransmittable_frames=*/true)) {
    // Fails if we try to write unencrypted stream data.
    delete frame->stream_frame;
    return false;
  }
}                                    

 Flush將數據發送到網絡中。

void QuicPacketCreator::Flush() {
  if (!HasPendingFrames()) {
    return;
  }

  // TODO(rtenneti): Change the default 64 alignas value (used the default
  // value from CACHELINE_SIZE).
  ALIGNAS(64) char seralized_packet_buffer[kMaxPacketSize];
  SerializePacket(seralized_packet_buffer, kMaxPacketSize);
  OnSerializedPacket();
}
void QuicConnection::OnSerializedPacket(SerializedPacket* serialized_packet) {
SendOrQueuePacket(serialized_packet);
}
void QuicConnection::SendOrQueuePacket(SerializedPacket* packet) {
  if (!queued_packets_.empty() || !WritePacket(packet)) {
    // Take ownership of the underlying encrypted packet.
    packet->encrypted_buffer = QuicUtils::CopyBuffer(*packet);
    queued_packets_.push_back(*packet);
    packet->retransmittable_frames.clear();
  }
}
bool QuicConnection::WritePacket(SerializedPacket* packet) {
  WriteResult result = writer_->WritePacket(
      packet->encrypted_buffer, encrypted_length, self_address().address(),
      peer_address(), per_packet_options_);
      //擁塞控制相關。
  bool reset_retransmission_alarm = sent_packet_manager_->OnPacketSent(
      packet, packet->original_path_id, packet->original_packet_number,
      packet_send_time, packet->transmission_type, IsRetransmittable(*packet));      
}

 新版的代碼，增加了send_buf_，數據包的處理過程有點繞。

[1] libquic https://github.com/devsisters/libquic