From 877b995a4d6a99ce280558f870f6c979080d8362 Mon Sep 17 00:00:00 2001 From: Francisco Date: Fri, 8 Oct 2021 18:05:38 +0100 Subject: [PATCH] gnb,nr: add metrics for ack'd TBS in the gNB. --- srsenb/hdr/stack/gnb_stack_nr.h | 4 ++- srsenb/hdr/stack/mac/nr/mac_nr.h | 8 +++++ srsenb/hdr/stack/mac/nr/sched_nr.h | 2 +- srsenb/hdr/stack/mac/nr/sched_nr_interface.h | 14 ++++---- srsenb/hdr/stack/mac/nr/sched_nr_ue.h | 4 +++ srsenb/hdr/stack/mac/nr/sched_nr_worker.h | 7 ++-- srsenb/src/stack/gnb_stack_nr.cc | 35 +++++++++++--------- srsenb/src/stack/mac/mac.cc | 8 ++--- srsenb/src/stack/mac/nr/mac_nr.cc | 33 ++++++++++++++++-- srsenb/src/stack/mac/nr/sched_nr.cc | 15 +++++++-- srsenb/src/stack/mac/nr/sched_nr_harq.cc | 2 +- srsenb/src/stack/mac/nr/sched_nr_worker.cc | 25 +++++++++++++- srsenb/test/mac/nr/sched_nr_test.cc | 4 +-- 13 files changed, 122 insertions(+), 39 deletions(-) diff --git a/srsenb/hdr/stack/gnb_stack_nr.h b/srsenb/hdr/stack/gnb_stack_nr.h index 0a48d1201..83fe9317f 100644 --- a/srsenb/hdr/stack/gnb_stack_nr.h +++ b/srsenb/hdr/stack/gnb_stack_nr.h @@ -123,7 +123,9 @@ private: srsran::task_multiqueue::queue_handle sync_task_queue, ue_task_queue, gtpu_task_queue, mac_task_queue, metrics_task_queue; - srsran::dyn_blocking_queue pending_stack_metrics; + // metrics waiting condition + std::mutex metrics_mutex; + std::condition_variable metrics_cvar; // derived srsenb::mac_nr mac; diff --git a/srsenb/hdr/stack/mac/nr/mac_nr.h b/srsenb/hdr/stack/mac/nr/mac_nr.h index 18bb2a0ee..08af3976a 100644 --- a/srsenb/hdr/stack/mac/nr/mac_nr.h +++ b/srsenb/hdr/stack/mac/nr/mac_nr.h @@ -82,6 +82,9 @@ private: // PDU processing int handle_pdu(srsran::unique_byte_buffer_t pdu); + // Metrics processing + void get_metrics_nolock(srsenb::mac_metrics_t& metrics); + // Encoding srsran::byte_buffer_t* assemble_rar(srsran::const_span grants); srsran::unique_byte_buffer_t rar_pdu_buffer = nullptr; @@ -124,6 +127,11 @@ private: // Number of rach preambles detected for a CC std::vector detected_rachs; + + // Metrics + std::mutex metrics_mutex; + std::condition_variable metrics_condvar; + srsenb::mac_metrics_t* metrics_pending = nullptr; }; } // namespace srsenb diff --git a/srsenb/hdr/stack/mac/nr/sched_nr.h b/srsenb/hdr/stack/mac/nr/sched_nr.h index 8608f7e00..659eb90cf 100644 --- a/srsenb/hdr/stack/mac/nr/sched_nr.h +++ b/srsenb/hdr/stack/mac/nr/sched_nr.h @@ -51,7 +51,7 @@ public: void ul_bsr(uint16_t rnti, uint32_t lcg_id, uint32_t bsr) override; void dl_buffer_state(uint16_t rnti, uint32_t lcid, uint32_t newtx, uint32_t retx); - int get_dl_sched(slot_point pdsch_tti, uint32_t cc, dl_sched_res_t& result) override; + int run_slot(slot_point pdsch_tti, uint32_t cc, dl_sched_res_t& result, mac_metrics_t* metrics = nullptr) override; int get_ul_sched(slot_point pusch_tti, uint32_t cc, ul_sched_t& result) override; private: diff --git a/srsenb/hdr/stack/mac/nr/sched_nr_interface.h b/srsenb/hdr/stack/mac/nr/sched_nr_interface.h index c0813aa5d..4b38a8bc5 100644 --- a/srsenb/hdr/stack/mac/nr/sched_nr_interface.h +++ b/srsenb/hdr/stack/mac/nr/sched_nr_interface.h @@ -33,6 +33,8 @@ const static size_t SCHED_NR_MAX_BWP_PER_CELL = 2; const static size_t SCHED_NR_MAX_LCID = 32; const static size_t SCHED_NR_MAX_LC_GROUP = 7; +struct mac_metrics_t; + class sched_nr_interface { public: @@ -119,12 +121,12 @@ public: }; virtual ~sched_nr_interface() = default; - virtual int config(const sched_cfg_t& sched_cfg, srsran::const_span ue_cfg) = 0; - virtual void ue_cfg(uint16_t rnti, const ue_cfg_t& ue_cfg) = 0; - virtual void ue_rem(uint16_t rnti) = 0; - virtual bool ue_exists(uint16_t rnti) = 0; - virtual int get_dl_sched(slot_point slot_rx, uint32_t cc, dl_sched_res_t& result) = 0; - virtual int get_ul_sched(slot_point slot_rx, uint32_t cc, ul_sched_t& result) = 0; + virtual int config(const sched_cfg_t& sched_cfg, srsran::const_span ue_cfg) = 0; + virtual void ue_cfg(uint16_t rnti, const ue_cfg_t& ue_cfg) = 0; + virtual void ue_rem(uint16_t rnti) = 0; + virtual bool ue_exists(uint16_t rnti) = 0; + virtual int run_slot(slot_point slot_rx, uint32_t cc, dl_sched_res_t& result, mac_metrics_t* metrics = nullptr) = 0; + virtual int get_ul_sched(slot_point slot_rx, uint32_t cc, ul_sched_t& result) = 0; virtual void dl_ack_info(uint16_t rnti, uint32_t cc, uint32_t pid, uint32_t tb_idx, bool ack) = 0; virtual void ul_crc_info(uint16_t rnti, uint32_t cc, uint32_t pid, bool crc) = 0; diff --git a/srsenb/hdr/stack/mac/nr/sched_nr_ue.h b/srsenb/hdr/stack/mac/nr/sched_nr_ue.h index 3887e23ff..b02d14db0 100644 --- a/srsenb/hdr/stack/mac/nr/sched_nr_ue.h +++ b/srsenb/hdr/stack/mac/nr/sched_nr_ue.h @@ -16,6 +16,7 @@ #include "sched_nr_cfg.h" #include "sched_nr_harq.h" #include "sched_nr_interface.h" +#include "srsenb/hdr/stack/mac/common/mac_metrics.h" #include "srsenb/hdr/stack/mac/common/ue_buffer_manager.h" #include "srsran/adt/circular_map.h" #include "srsran/adt/move_callback.h" @@ -73,6 +74,9 @@ public: harq_entity harq_ent; + // metrics + mac_ue_metrics_t metrics = {}; + private: bwp_ue_cfg bwp_cfg; const sched_cell_params& cell_params; diff --git a/srsenb/hdr/stack/mac/nr/sched_nr_worker.h b/srsenb/hdr/stack/mac/nr/sched_nr_worker.h index d87404002..47ba12c8b 100644 --- a/srsenb/hdr/stack/mac/nr/sched_nr_worker.h +++ b/srsenb/hdr/stack/mac/nr/sched_nr_worker.h @@ -25,6 +25,9 @@ #include namespace srsenb { + +struct mac_metrics_t; + namespace sched_nr_impl { using dl_sched_t = sched_nr_interface::dl_sched_t; @@ -92,7 +95,7 @@ public: sched_worker_manager(sched_worker_manager&&) = delete; ~sched_worker_manager(); - void run_slot(slot_point slot_tx, uint32_t cc, dl_sched_res_t& dl_res, ul_sched_t& ul_res); + void run_slot(slot_point slot_tx, uint32_t cc, dl_sched_res_t& dl_res, ul_sched_t& ul_res, mac_metrics_t* metrics); void enqueue_event(uint16_t rnti, srsran::move_callback ev); void enqueue_cc_event(uint32_t cc, srsran::move_callback ev); @@ -103,7 +106,7 @@ public: private: void update_ue_db(slot_point slot_tx, bool locked_context); - + void get_metrics(mac_metrics_t& metrics); bool save_sched_result(slot_point pdcch_slot, uint32_t cc, dl_sched_res_t& dl_res, ul_sched_t& ul_res); const sched_params& cfg; diff --git a/srsenb/src/stack/gnb_stack_nr.cc b/srsenb/src/stack/gnb_stack_nr.cc index 62016bd32..417f936a9 100644 --- a/srsenb/src/stack/gnb_stack_nr.cc +++ b/srsenb/src/stack/gnb_stack_nr.cc @@ -25,7 +25,6 @@ gnb_stack_nr::gnb_stack_nr(srslog::sink& log_sink) : pdcp_logger(srslog::fetch_basic_logger("PDCP-NR", log_sink, false)), rrc_logger(srslog::fetch_basic_logger("RRC-NR", log_sink, false)), stack_logger(srslog::fetch_basic_logger("STCK-NR", log_sink, false)), - pending_stack_metrics(64), mac(&task_sched), rrc(&task_sched), pdcp(&task_sched, pdcp_logger), @@ -54,7 +53,7 @@ int gnb_stack_nr::init(const gnb_stack_args_t& args_, x2_interface* x2_) { args = args_; - phy = phy_; + phy = phy_; // setup logging mac_logger.set_level(srslog::str_to_basic_level(args.log.mac_level)); @@ -141,22 +140,28 @@ void gnb_stack_nr::process_pdus() {} bool gnb_stack_nr::get_metrics(srsenb::stack_metrics_t* metrics) { - // use stack thread to query metrics - auto ret = metrics_task_queue.try_push([this]() { - srsenb::stack_metrics_t metrics{}; - mac.get_metrics(metrics.mac); - rrc.get_metrics(metrics.rrc); - if (not pending_stack_metrics.try_push(std::move(metrics))) { - stack_logger.error("Unable to push metrics to queue"); + bool metrics_ready = false; + + // use stack thread to query RRC metrics + auto ret = metrics_task_queue.try_push([this, metrics, &metrics_ready]() { + rrc.get_metrics(metrics->rrc); + { + std::lock_guard lock(metrics_mutex); + metrics_ready = true; } + metrics_cvar.notify_one(); }); - - if (ret.has_value()) { - // wait for result - *metrics = pending_stack_metrics.pop_blocking(); - return true; + if (not ret.has_value()) { + return false; } - return false; + + // obtain MAC metrics (do not use stack thread) + mac.get_metrics(metrics->mac); + + // wait for RRC result + std::unique_lock lock(metrics_mutex); + metrics_cvar.wait(lock, [&metrics_ready]() { return metrics_ready; }); + return true; } // Temporary GW interface diff --git a/srsenb/src/stack/mac/mac.cc b/srsenb/src/stack/mac/mac.cc index 8e66ef59d..88c919f8f 100644 --- a/srsenb/src/stack/mac/mac.cc +++ b/srsenb/src/stack/mac/mac.cc @@ -588,7 +588,7 @@ int mac::get_dl_sched(uint32_t tti_tx_dl, dl_sched_list_t& dl_sched_res_list) return 0; } - trace_threshold_complete_event("mac::get_dl_sched", "total_time", std::chrono::microseconds(100)); + trace_threshold_complete_event("mac::run_slot", "total_time", std::chrono::microseconds(100)); logger.set_context(TTI_SUB(tti_tx_dl, FDD_HARQ_DELAY_UL_MS)); if (do_padding) { add_padding(); @@ -832,9 +832,9 @@ int mac::get_mch_sched(uint32_t tti, bool is_mcch, dl_sched_list_t& dl_sched_res int requested_bytes = (mcs_data.tbs / 8 > (int)mch.mtch_sched[mtch_index].lcid_buffer_size) ? (mch.mtch_sched[mtch_index].lcid_buffer_size) : ((mcs_data.tbs / 8) - 2); - int bytes_received = ue_db[SRSRAN_MRNTI]->read_pdu(current_lcid, mtch_payload_buffer, requested_bytes); - mch.pdu[0].lcid = current_lcid; - mch.pdu[0].nbytes = bytes_received; + int bytes_received = ue_db[SRSRAN_MRNTI]->read_pdu(current_lcid, mtch_payload_buffer, requested_bytes); + mch.pdu[0].lcid = current_lcid; + mch.pdu[0].nbytes = bytes_received; mch.mtch_sched[0].mtch_payload = mtch_payload_buffer; dl_sched_res->pdsch[0].dci.rnti = SRSRAN_MRNTI; if (bytes_received) { diff --git a/srsenb/src/stack/mac/nr/mac_nr.cc b/srsenb/src/stack/mac/nr/mac_nr.cc index 30654c232..edeb2ac1c 100644 --- a/srsenb/src/stack/mac/nr/mac_nr.cc +++ b/srsenb/src/stack/mac/nr/mac_nr.cc @@ -76,7 +76,18 @@ void mac_nr::stop() } } +/// Called from metrics thread void mac_nr::get_metrics(srsenb::mac_metrics_t& metrics) +{ + // Requests asynchronously MAC metrics + std::unique_lock lock(metrics_mutex); + metrics_pending = &metrics; + + // Blocks waiting for results + metrics_condvar.wait(lock, [this]() { return metrics_pending == nullptr; }); +} + +void mac_nr::get_metrics_nolock(srsenb::mac_metrics_t& metrics) { srsran::rwlock_read_guard lock(rwlock); metrics.ues.reserve(ue_db.size()); @@ -274,9 +285,25 @@ int mac_nr::get_dl_sched(const srsran_slot_cfg_t& slot_cfg, dl_sched_t& dl_sched slot_point pdsch_slot = srsran::slot_point{NUMEROLOGY_IDX, slot_cfg.idx}; sched_nr_interface::dl_sched_res_t dl_res; - int ret = sched.get_dl_sched(pdsch_slot, 0, dl_res); - if (ret != SRSRAN_SUCCESS) { - return ret; + + // Get metrics if requested + { + std::unique_lock metrics_lock(metrics_mutex); + if (metrics_pending != nullptr) { + get_metrics_nolock(*metrics_pending); + } + + // Run Scheduler + int ret = sched.run_slot(pdsch_slot, 0, dl_res, metrics_pending); + + // Notify metrics are filled, if requested + if (metrics_pending != nullptr) { + metrics_pending = nullptr; + metrics_condvar.notify_one(); + } + if (ret != SRSRAN_SUCCESS) { + return ret; + } } dl_sched = dl_res.dl_sched; diff --git a/srsenb/src/stack/mac/nr/sched_nr.cc b/srsenb/src/stack/mac/nr/sched_nr.cc index 270bc998c..827b4bb85 100644 --- a/srsenb/src/stack/mac/nr/sched_nr.cc +++ b/srsenb/src/stack/mac/nr/sched_nr.cc @@ -11,6 +11,7 @@ */ #include "srsenb/hdr/stack/mac/nr/sched_nr.h" +#include "srsenb/hdr/stack/mac/common/mac_metrics.h" #include "srsenb/hdr/stack/mac/nr/harq_softbuffer.h" #include "srsenb/hdr/stack/mac/nr/sched_nr_cell.h" #include "srsenb/hdr/stack/mac/nr/sched_nr_worker.h" @@ -123,13 +124,13 @@ void sched_nr::ue_cfg_impl(uint16_t rnti, const ue_cfg_t& uecfg) } /// Generate {pdcch_slot,cc} scheduling decision -int sched_nr::get_dl_sched(slot_point slot_dl, uint32_t cc, dl_sched_res_t& result) +int sched_nr::run_slot(slot_point slot_dl, uint32_t cc, dl_sched_res_t& result, mac_metrics_t* metrics) { // Copy UL results to intermediate buffer ul_sched_t& ul_res = pending_results->add_ul_result(slot_dl, cc); // Generate {slot_idx,cc} result - sched_workers->run_slot(slot_dl, cc, result, ul_res); + sched_workers->run_slot(slot_dl, cc, result, ul_res, metrics); return SRSRAN_SUCCESS; } @@ -156,7 +157,15 @@ int sched_nr::dl_rach_info(uint32_t cc, const dl_sched_rar_info_t& rar_info) void sched_nr::dl_ack_info(uint16_t rnti, uint32_t cc, uint32_t pid, uint32_t tb_idx, bool ack) { sched_workers->enqueue_cc_feedback(rnti, cc, [this, pid, tb_idx, ack](ue_carrier& ue_cc) { - if (ue_cc.harq_ent.dl_ack_info(pid, tb_idx, ack) != SRSRAN_SUCCESS) { + int tbs = ue_cc.harq_ent.dl_ack_info(pid, tb_idx, ack); + if (tbs >= 0) { + if (ack) { + ue_cc.metrics.tx_brate += tbs * 8; + } else { + ue_cc.metrics.tx_errors++; + } + ue_cc.metrics.tx_pkts++; + } else { logger->warning("SCHED: rnti=0x%x, received DL HARQ-ACK for empty pid=%d", ue_cc.rnti, pid); } }); diff --git a/srsenb/src/stack/mac/nr/sched_nr_harq.cc b/srsenb/src/stack/mac/nr/sched_nr_harq.cc index 19800cc0f..0b184f38f 100644 --- a/srsenb/src/stack/mac/nr/sched_nr_harq.cc +++ b/srsenb/src/stack/mac/nr/sched_nr_harq.cc @@ -25,7 +25,7 @@ int harq_proc::ack_info(uint32_t tb_idx, bool ack) if (ack) { tb[tb_idx].active = false; } - return SRSRAN_SUCCESS; + return tb[tb_idx].tbs; } void harq_proc::new_slot(slot_point slot_rx) diff --git a/srsenb/src/stack/mac/nr/sched_nr_worker.cc b/srsenb/src/stack/mac/nr/sched_nr_worker.cc index d2f0fe44e..8b533511a 100644 --- a/srsenb/src/stack/mac/nr/sched_nr_worker.cc +++ b/srsenb/src/stack/mac/nr/sched_nr_worker.cc @@ -11,6 +11,7 @@ */ #include "srsenb/hdr/stack/mac/nr/sched_nr_worker.h" +#include "srsenb/hdr/stack/mac/common/mac_metrics.h" #include "srsenb/hdr/stack/mac/nr/sched_nr_signalling.h" #include "srsran/common/string_helpers.h" @@ -255,7 +256,11 @@ void sched_worker_manager::update_ue_db(slot_point slot_tx, bool locked_context) } } -void sched_worker_manager::run_slot(slot_point slot_tx, uint32_t cc, dl_sched_res_t& dl_res, ul_sched_t& ul_res) +void sched_worker_manager::run_slot(slot_point slot_tx, + uint32_t cc, + dl_sched_res_t& dl_res, + ul_sched_t& ul_res, + mac_metrics_t* metrics) { // Fill DL signalling messages that do not depend on UEs state serv_cell_manager& serv_cell = *cells[cc]; @@ -284,6 +289,11 @@ void sched_worker_manager::run_slot(slot_point slot_tx, uint32_t cc, dl_sched_re } update_ue_db(slot_tx, true); + // Obtain MAC metrics if requested + if (metrics != nullptr) { + get_metrics(*metrics); + } + // mark the start of slot. awake remaining workers if locking on the mutex current_slot = slot_tx; worker_count.store(static_cast(cc_worker_list.size()), std::memory_order_relaxed); @@ -359,5 +369,18 @@ bool sched_worker_manager::save_sched_result(slot_point pdcch_slot, return true; } +void sched_worker_manager::get_metrics(mac_metrics_t& metrics) +{ + for (mac_ue_metrics_t& ue_metric : metrics.ues) { + if (ue_db.contains(ue_metric.rnti) and ue_db[ue_metric.rnti]->carriers[0] != nullptr) { + auto& ue_cc = *ue_db[ue_metric.rnti]->carriers[0]; + ue_metric.tx_brate = ue_cc.metrics.tx_brate; + ue_metric.tx_errors = ue_cc.metrics.tx_errors; + ue_metric.tx_pkts = ue_cc.metrics.tx_pkts; + ue_cc.metrics = {}; + } + } +} + } // namespace sched_nr_impl } // namespace srsenb diff --git a/srsenb/test/mac/nr/sched_nr_test.cc b/srsenb/test/mac/nr/sched_nr_test.cc index 8b79a3edb..51f995f6d 100644 --- a/srsenb/test/mac/nr/sched_nr_test.cc +++ b/srsenb/test/mac/nr/sched_nr_test.cc @@ -103,7 +103,7 @@ void sched_nr_cfg_serialized_test() sched_nr_interface::dl_sched_res_t dl_res; sched_nr_interface::ul_sched_t ul_res; auto tp1 = std::chrono::steady_clock::now(); - TESTASSERT(sched_tester.get_sched()->get_dl_sched(slot_tx, cc, dl_res) == SRSRAN_SUCCESS); + TESTASSERT(sched_tester.get_sched()->run_slot(slot_tx, cc, dl_res) == SRSRAN_SUCCESS); TESTASSERT(sched_tester.get_sched()->get_ul_sched(slot_tx, cc, ul_res) == SRSRAN_SUCCESS); auto tp2 = std::chrono::steady_clock::now(); count_per_cc[cc] += std::chrono::duration_cast(tp2 - tp1).count(); @@ -155,7 +155,7 @@ void sched_nr_cfg_parallel_cc_test() sched_nr_interface::dl_sched_res_t dl_res; sched_nr_interface::ul_sched_t ul_res; auto tp1 = std::chrono::steady_clock::now(); - TESTASSERT(sched_tester.get_sched()->get_dl_sched(slot_tx, cc, dl_res) == SRSRAN_SUCCESS); + TESTASSERT(sched_tester.get_sched()->run_slot(slot_tx, cc, dl_res) == SRSRAN_SUCCESS); TESTASSERT(sched_tester.get_sched()->get_ul_sched(slot_tx, cc, ul_res) == SRSRAN_SUCCESS); auto tp2 = std::chrono::steady_clock::now(); nano_count[cc].fetch_add(std::chrono::duration_cast(tp2 - tp1).count(),