diff --git a/lib/include/srsran/upper/rlc_am_lte.h b/lib/include/srsran/upper/rlc_am_lte.h index 052a82fee..cef259475 100644 --- a/lib/include/srsran/upper/rlc_am_lte.h +++ b/lib/include/srsran/upper/rlc_am_lte.h @@ -16,6 +16,7 @@ #include "srsran/adt/accumulators.h" #include "srsran/adt/circular_array.h" #include "srsran/adt/circular_map.h" +#include "srsran/adt/intrusive_list.h" #include "srsran/common/buffer_pool.h" #include "srsran/common/common.h" #include "srsran/common/srsran_assert.h" @@ -33,94 +34,101 @@ namespace srsran { #undef RLC_AM_BUFFER_DEBUG -/// RLC AM PDU Segment, containing the PDCP SN and RLC SN it has been assigned to, and its current ACK state -struct rlc_am_pdu_segment { - const static uint32_t invalid_sn = std::numeric_limits::max(); - - bool empty() const { return rlc_sn() == invalid_sn and pdcp_sn() == invalid_sn; } - void set_ack(bool val = true) { acked = val; } - bool is_acked() const { return acked; } - uint32_t rlc_sn() const { return rlc_sn_; } - uint32_t pdcp_sn() const { return pdcp_sn_; } - -protected: - uint32_t rlc_sn_ = invalid_sn; - uint32_t pdcp_sn_ = invalid_sn; - bool acked = false; -}; - -template -struct pdu_segment_list; -using rlc_pdu_segment_list = pdu_segment_list; -using pdcp_pdu_segment_list = pdu_segment_list; +struct pdcp_pdu_segment_list; +struct rlc_pdu_segment_list; -/// Pool that manages the allocation of RLC AM PDU Segments to RLC SDUs +/// Pool that manages the allocation of RLC AM PDU Segments to RLC PDUs and tracking of segments ACK state struct rlc_am_pdu_segment_pool { const static size_t MAX_POOL_SIZE = 16384; + using rlc_list_tag = default_intrusive_tag; + struct free_list_tag {}; + + /// RLC AM PDU Segment, containing the PDCP SN and RLC SN it has been assigned to, and its current ACK state + struct segment_resource : public intrusive_forward_list_element, + public intrusive_forward_list_element, + public intrusive_double_linked_list_element<> { + const static uint32_t invalid_sn = std::numeric_limits::max(); + + int id() const; + void release_pdcp_sn(); + void release_rlc_sn(); + uint32_t rlc_sn() const { return rlc_sn_; } + uint32_t pdcp_sn() const { return pdcp_sn_; } + bool empty() const { return rlc_sn_ == invalid_sn and pdcp_sn_ == invalid_sn; } + bool is_acked() const { return rlc_sn() != invalid_sn; } - struct segment_resource : public rlc_am_pdu_segment { - int id() const; - void deallocate(); - - using rlc_am_pdu_segment::pdcp_sn_; - using rlc_am_pdu_segment::rlc_sn_; - - // intrusive same RLC PDU segment list - segment_resource* rlc_next = nullptr; - // intrusive same PDCP PDU segment list - segment_resource* pdcp_next = nullptr; - - // intrusive linked lists - segment_resource* next_free = nullptr; + private: + friend struct rlc_am_pdu_segment_pool; + uint32_t rlc_sn_ = invalid_sn; + uint32_t pdcp_sn_ = invalid_sn; rlc_am_pdu_segment_pool* parent_pool = nullptr; }; rlc_am_pdu_segment_pool(); rlc_am_pdu_segment_pool(const rlc_am_pdu_segment_pool&) = delete; rlc_am_pdu_segment_pool(rlc_am_pdu_segment_pool&&) = delete; - bool has_segments() const; + bool has_segments() const { return not free_list.empty(); } bool - allocate_segment(uint32_t rlc_sn, rlc_pdu_segment_list& rlc_list, uint32_t pdcp_sn, pdcp_pdu_segment_list& pdcp_list); + make_segment(uint32_t rlc_sn, uint32_t pdcp_sn, rlc_pdu_segment_list& rlc_list, pdcp_pdu_segment_list& pdcp_list); private: - segment_resource* free_list = nullptr; - std::array segments; + intrusive_forward_list free_list; + std::array segments; }; -template -struct pdu_segment_list { - void push(rlc_am_pdu_segment_pool::segment_resource* obj); - void clear() { head.reset(); } - - struct iterator : public std::iterator { - explicit iterator(rlc_am_pdu_segment_pool::segment_resource* item_ = nullptr) : item(item_) {} - const rlc_am_pdu_segment* operator->() const { return item; } - rlc_am_pdu_segment* operator->() { return item; } - const rlc_am_pdu_segment& operator*() const { return *item; } - rlc_am_pdu_segment& operator*() { return *item; } - iterator& operator++() - { - item = (rlcSnList) ? item->rlc_next : item->pdcp_next; - return *this; - } - bool operator==(iterator other) const { return item == other.item; } - bool operator!=(iterator other) const { return item != other.item; } +/// RLC AM PDU Segment, containing the PDCP SN and RLC SN it has been assigned to, and its current ACK state +using rlc_am_pdu_segment = rlc_am_pdu_segment_pool::segment_resource; - private: - rlc_am_pdu_segment_pool::segment_resource* item; - }; - using const_iterator = iterator; +class pdcp_pdu_segment_list +{ + using list_type = intrusive_double_linked_list; + list_type list; - iterator begin() { return iterator(head.get()); } - iterator end() { return iterator(nullptr); } - const_iterator begin() const { return iterator(head.get()); } - const_iterator end() const { return iterator(nullptr); } +public: + using iterator = typename list_type::iterator; + using const_iterator = typename list_type::const_iterator; + + iterator begin() { return list.begin(); } + iterator end() { return list.end(); } + const_iterator begin() const { return list.begin(); } + const_iterator end() const { return list.end(); } + bool empty() const { return list.empty(); } + + pdcp_pdu_segment_list() = default; + pdcp_pdu_segment_list(pdcp_pdu_segment_list&&) noexcept = default; + pdcp_pdu_segment_list& operator=(pdcp_pdu_segment_list&&) noexcept = default; + ~pdcp_pdu_segment_list() { clear(); } + void push(rlc_am_pdu_segment& segment) { list.push_front(&segment); } + void pop(rlc_am_pdu_segment& segment); + void clear() + { + while (not empty()) { + pop(*begin()); + } + } +}; + +struct rlc_pdu_segment_list { + using list_type = intrusive_forward_list; + using iterator = typename list_type::iterator; + using const_iterator = typename list_type::const_iterator; + + const_iterator begin() const { return list.begin(); } + const_iterator end() const { return list.end(); } + iterator begin() { return list.begin(); } + iterator end() { return list.end(); } + bool empty() const { return list.empty(); } + + rlc_pdu_segment_list() = default; + rlc_pdu_segment_list(rlc_pdu_segment_list&& other) noexcept = default; + rlc_pdu_segment_list(const rlc_pdu_segment_list&) = delete; + rlc_pdu_segment_list& operator=(rlc_pdu_segment_list&& other) noexcept = default; + ~rlc_pdu_segment_list() { clear(); } + void push(rlc_am_pdu_segment& segment) { list.push_front(&segment); } + void clear(); private: - struct list_deleter { - void operator()(rlc_am_pdu_segment_pool::segment_resource* ptr); - }; - std::unique_ptr head; + list_type list; }; // @@ -136,12 +144,12 @@ struct rlc_amd_rx_pdu_segments_t { }; struct rlc_amd_tx_pdu_t { - rlc_amd_pdu_header_t header; - unique_byte_buffer_t buf; - pdcp_pdu_segment_list pdcp_sn_list; - uint32_t retx_count; - uint32_t rlc_sn = std::numeric_limits::max(); - bool is_acked = false; + rlc_amd_pdu_header_t header; + unique_byte_buffer_t buf; + rlc_pdu_segment_list segment_list; + uint32_t retx_count; + uint32_t rlc_sn = std::numeric_limits::max(); + bool is_acked = false; }; struct rlc_amd_retx_t { @@ -157,11 +165,10 @@ struct rlc_sn_info_t { }; struct pdcp_sdu_info_t { - uint32_t sn; - bool fully_txed; // Boolean indicating if the SDU is fully transmitted. - bool fully_acked; // Boolean indicating if the SDU is fully acked. This is only necessary temporarely to avoid - // duplicate removal from the queue while processing the status report - rlc_pdu_segment_list rlc_segment_list; // List of RLC PDUs in transit and whether they have been acked or not. + uint32_t sn; + bool fully_txed; // Boolean indicating if the SDU is fully transmitted. + pdcp_pdu_segment_list segment_list; // List of RLC PDUs in transit and whether they have been acked or not. + bool fully_acked() const { return segment_list.empty(); } }; template @@ -222,10 +229,9 @@ public: if (buffered_pdus[sn_idx].sn == invalid_sn) { return; } - buffered_pdus[sn_idx].sn = invalid_sn; - buffered_pdus[sn_idx].fully_acked = false; - buffered_pdus[sn_idx].fully_txed = false; - buffered_pdus[sn_idx].rlc_segment_list.clear(); + buffered_pdus[sn_idx].sn = invalid_sn; + buffered_pdus[sn_idx].fully_txed = false; + buffered_pdus[sn_idx].segment_list.clear(); count--; } diff --git a/lib/src/upper/rlc_am_lte.cc b/lib/src/upper/rlc_am_lte.cc index 2f8a6768b..ab650a440 100644 --- a/lib/src/upper/rlc_am_lte.cc +++ b/lib/src/upper/rlc_am_lte.cc @@ -16,7 +16,6 @@ #include "srsran/interfaces/ue_rrc_interfaces.h" #include "srsran/srslog/event_trace.h" #include -#include #define MOD 1024 #define RX_MOD_BASE(x) (((x)-vr_r) % 1024) @@ -67,75 +66,61 @@ int rlc_am_pdu_segment_pool::segment_resource::id() const return std::distance(parent_pool->segments.cbegin(), this); } -void rlc_am_pdu_segment_pool::segment_resource::deallocate() +void rlc_am_pdu_segment_pool::segment_resource::release_pdcp_sn() { - acked = false; - next_free = parent_pool->free_list; - parent_pool->free_list = this; + pdcp_sn_ = invalid_sn; + if (empty()) { + parent_pool->free_list.push_front(this); + } } -rlc_am_pdu_segment_pool::rlc_am_pdu_segment_pool() +void rlc_am_pdu_segment_pool::segment_resource::release_rlc_sn() { - for (segment_resource& s : segments) { - s.parent_pool = this; - s.next_free = free_list; - free_list = &s; + rlc_sn_ = invalid_sn; + if (empty()) { + parent_pool->free_list.push_front(this); } } -bool rlc_am_pdu_segment_pool::has_segments() const +rlc_am_pdu_segment_pool::rlc_am_pdu_segment_pool() { - return free_list != nullptr; + for (segment_resource& s : segments) { + s.parent_pool = this; + free_list.push_front(&s); + } } -bool rlc_am_pdu_segment_pool::allocate_segment(uint32_t rlc_sn, - rlc_pdu_segment_list& rlc_list, - uint32_t pdcp_sn, - pdcp_pdu_segment_list& pdcp_list) +bool rlc_am_pdu_segment_pool::make_segment(uint32_t rlc_sn, + uint32_t pdcp_sn, + rlc_pdu_segment_list& rlc_list, + pdcp_pdu_segment_list& pdcp_list) { - if (free_list == nullptr) { + if (not has_segments()) { return false; } - segment_resource* segment = free_list; - free_list = segment->next_free; - rlc_list.push(segment); - segment->rlc_sn_ = rlc_sn; - pdcp_list.push(segment); + segment_resource* segment = free_list.pop_front(); + rlc_list.push(*segment); + pdcp_list.push(*segment); + segment->rlc_sn_ = rlc_sn; segment->pdcp_sn_ = pdcp_sn; return true; } -template -void pdu_segment_list::push(rlc_am_pdu_segment_pool::segment_resource* obj) +void pdcp_pdu_segment_list::pop(rlc_am_pdu_segment& segment) { - if (head != nullptr) { - if (rlcSnList) { - obj->rlc_next = head.release(); - } else { - obj->pdcp_next = head.release(); - } - } - head.reset(obj); + // remove from list + list.pop(&segment); + // signal pool that the pdcp handle is released + segment.release_pdcp_sn(); } -template -void pdu_segment_list::list_deleter::operator()(rlc_am_pdu_segment_pool::segment_resource* node) +void rlc_pdu_segment_list::clear() { - while (node != nullptr) { - rlc_am_pdu_segment_pool::segment_resource* next = nullptr; - if (rlcSnList) { - next = node->rlc_next; - node->rlc_next = nullptr; - node->rlc_sn_ = rlc_am_pdu_segment::invalid_sn; - } else { - next = node->pdcp_next; - node->pdcp_next = nullptr; - node->pdcp_sn_ = rlc_am_pdu_segment::invalid_sn; - } - if (node->empty()) { - node->deallocate(); - } - node = next; + while (not empty()) { + // remove from list + rlc_am_pdu_segment* segment = list.pop_front(); + // deallocate if also removed from PDCP + segment->release_rlc_sn(); } } @@ -417,7 +402,7 @@ void rlc_am_lte::rlc_am_lte_tx::check_sn_reached_max_retx(uint32_t sn) logger.warning("%s Signaling max number of reTx=%d for SN=%d", RB_NAME, tx_window[sn].retx_count, sn); parent->rrc->max_retx_attempted(); srsran::pdcp_sn_vector_t pdcp_sns; - for (const rlc_am_pdu_segment& segment : tx_window[sn].pdcp_sn_list) { + for (const rlc_am_pdu_segment& segment : tx_window[sn].segment_list) { pdcp_sns.push_back(segment.pdcp_sn()); } parent->pdcp->notify_failure(parent->lcid, pdcp_sns); @@ -1043,7 +1028,7 @@ int rlc_am_lte::rlc_am_lte_tx::build_data_pdu(uint8_t* payload, uint32_t nof_byt tx_sdu->msg += to_move; if (undelivered_sdu_info_queue.has_pdcp_sn(tx_sdu->md.pdcp_sn)) { pdcp_sdu_info_t& pdcp_sdu = undelivered_sdu_info_queue[tx_sdu->md.pdcp_sn]; - segment_pool.allocate_segment(header.sn, pdcp_sdu.rlc_segment_list, tx_sdu->md.pdcp_sn, tx_pdu.pdcp_sn_list); + segment_pool.make_segment(header.sn, tx_sdu->md.pdcp_sn, tx_pdu.segment_list, pdcp_sdu.segment_list); if (tx_sdu->N_bytes == 0) { pdcp_sdu.fully_txed = true; } @@ -1115,7 +1100,7 @@ int rlc_am_lte::rlc_am_lte_tx::build_data_pdu(uint8_t* payload, uint32_t nof_byt tx_sdu->msg += to_move; if (undelivered_sdu_info_queue.has_pdcp_sn(tx_sdu->md.pdcp_sn)) { pdcp_sdu_info_t& pdcp_sdu = undelivered_sdu_info_queue[tx_sdu->md.pdcp_sn]; - segment_pool.allocate_segment(header.sn, pdcp_sdu.rlc_segment_list, tx_sdu->md.pdcp_sn, tx_pdu.pdcp_sn_list); + segment_pool.make_segment(header.sn, tx_sdu->md.pdcp_sn, tx_pdu.segment_list, pdcp_sdu.segment_list); if (tx_sdu->N_bytes == 0) { pdcp_sdu.fully_txed = true; } @@ -1321,28 +1306,25 @@ void rlc_am_lte::rlc_am_lte_tx::update_notification_ack_info(const rlc_amd_tx_pd if (not tx_window.has_sn(tx_pdu.header.sn)) { return; } - pdcp_pdu_segment_list& pdcp_sns = tx_window[tx_pdu.header.sn].pdcp_sn_list; - for (rlc_am_pdu_segment& pdcp_segment : pdcp_sns) { - // Iterate over all SNs that were TX'ed - uint32_t pdcp_sn = pdcp_segment.pdcp_sn(); - pdcp_segment.set_ack(); + rlc_pdu_segment_list& pdcp_sns = tx_window[tx_pdu.header.sn].segment_list; + // Iterate over all PDCP SNs of the same RLC PDU that were TX'ed + for (rlc_am_pdu_segment& acked_segment : pdcp_sns) { + uint32_t pdcp_sn = acked_segment.pdcp_sn(); + pdcp_sdu_info_t& info = undelivered_sdu_info_queue[pdcp_sn]; - pdcp_sdu_info_t& info = undelivered_sdu_info_queue[pdcp_sn]; - // Check wether the SDU was fully acked - if (info.fully_txed and not info.fully_acked) { + // Remove RLC SN from PDCP PDU undelivered list + info.segment_list.pop(acked_segment); + + // Check whether the SDU was fully acked + if (info.fully_txed and info.segment_list.empty()) { // Check if all SNs were ACK'ed - info.fully_acked = std::all_of(info.rlc_segment_list.begin(), - info.rlc_segment_list.end(), - [](const rlc_am_pdu_segment& elem) { return elem.is_acked(); }); - if (info.fully_acked) { - if (not notify_info_vec.full()) { - notify_info_vec.push_back(pdcp_sn); - } else { - logger.warning("Can't notify delivery of PDCP_SN=%d.", pdcp_sn); - } - logger.debug("Erasing SDU info: PDCP_SN=%d", pdcp_sn); - undelivered_sdu_info_queue.clear_pdcp_sdu(pdcp_sn); + if (not notify_info_vec.full()) { + notify_info_vec.push_back(pdcp_sn); + } else { + logger.warning("Can't notify delivery of PDCP_SN=%d.", pdcp_sn); } + logger.debug("Erasing SDU info: PDCP_SN=%d", pdcp_sn); + undelivered_sdu_info_queue.clear_pdcp_sdu(pdcp_sn); } } pdcp_sns.clear(); @@ -2208,10 +2190,9 @@ void buffered_pdcp_pdu_list::clear() { count = 0; for (auto& b : buffered_pdus) { - b.sn = invalid_sn; - b.fully_acked = false; - b.fully_txed = false; - b.rlc_segment_list.clear(); + b.sn = invalid_sn; + b.fully_txed = false; + b.segment_list.clear(); } } @@ -2475,9 +2456,9 @@ void rlc_am_undelivered_sdu_info_to_string(fmt::memory_buffer& buffer, const std { fmt::format_to(buffer, "\n"); for (const auto& pdcp_pdu : info_queue) { - fmt::format_to(buffer, "\tPDCP_SN = {}, RLC_SNs = [", pdcp_pdu.sn); - for (const auto& rlc_sn_info : pdcp_pdu.rlc_segment_list) { - fmt::format_to(buffer, "{}ACK={}, ", rlc_sn_info.is_acked() ? "" : "N", rlc_sn_info.rlc_sn()); + fmt::format_to(buffer, "\tPDCP_SN = {}, undelivered RLC SNs = [", pdcp_pdu.sn); + for (const auto& rlc_sn_info : pdcp_pdu.segment_list) { + fmt::format_to(buffer, "{} ", rlc_sn_info.rlc_sn()); } fmt::format_to(buffer, "]\n"); }