From bc4264277a8db7e636f9462fc0e32a50e702bc3d Mon Sep 17 00:00:00 2001 From: Francisco Date: Thu, 14 Oct 2021 15:17:03 +0100 Subject: [PATCH] sched,nr: update NR scheduler tester to support parallel CC workers --- srsenb/test/mac/nr/sched_nr_sim_ue.cc | 181 ++++++++++++-------------- srsenb/test/mac/nr/sched_nr_sim_ue.h | 104 ++++----------- srsenb/test/mac/nr/sched_nr_test.cc | 102 +++++---------- 3 files changed, 138 insertions(+), 249 deletions(-) diff --git a/srsenb/test/mac/nr/sched_nr_sim_ue.cc b/srsenb/test/mac/nr/sched_nr_sim_ue.cc index 39f45222d..1f4ba8b3b 100644 --- a/srsenb/test/mac/nr/sched_nr_sim_ue.cc +++ b/srsenb/test/mac/nr/sched_nr_sim_ue.cc @@ -14,6 +14,7 @@ #include "sched_nr_common_test.h" #include "sched_nr_ue_ded_test_suite.h" #include "srsran/common/test_common.h" +#include "srsran/common/thread_pool.h" namespace srsenb { @@ -90,41 +91,63 @@ void sched_nr_ue_sim::update_dl_harqs(const sched_nr_cc_result_view& cc_out) } } -namespace detail { - -sched_nr_sim_base::sched_nr_sim_base(const sched_nr_interface::sched_args_t& sched_args, - const std::vector& cell_cfg_list, - std::string test_name_) : +sched_nr_base_tester::sched_nr_base_tester(const sched_nr_interface::sched_args_t& sched_args, + const std::vector& cell_cfg_list, + std::string test_name_, + uint32_t nof_workers) : logger(srslog::fetch_basic_logger("TEST")), mac_logger(srslog::fetch_basic_logger("MAC")), sched_ptr(new sched_nr()), test_name(std::move(test_name_)) { - logger.info("\n=========== Start %s ===========", test_name.c_str()); + sem_init(&slot_sem, 0, 1); + + printf("\n=========== Start %s ===========\n", test_name.c_str()); cell_params.reserve(cell_cfg_list.size()); for (uint32_t cc = 0; cc < cell_cfg_list.size(); ++cc) { cell_params.emplace_back(cc, cell_cfg_list[cc], sched_args); } sched_ptr->config(sched_args, cell_cfg_list); // call parent cfg + cc_workers.resize(nof_workers - 1); + for (uint32_t i = 0; i < cc_workers.size(); ++i) { + fmt::memory_buffer fmtbuf; + fmt::format_to(fmtbuf, "worker{}", i + 1); + cc_workers[i].reset(new srsran::task_worker{to_string(fmtbuf), 10}); + } + cc_results.resize(cell_params.size()); TESTASSERT(cell_params.size() > 0); } -sched_nr_sim_base::~sched_nr_sim_base() +sched_nr_base_tester::~sched_nr_base_tester() { - logger.info("=========== End %s ==========\n", test_name.c_str()); + stop(); +} + +void sched_nr_base_tester::stop() +{ + bool stopping = not stopped.exchange(true); + if (stopping) { + sem_wait(&slot_sem); + sem_post(&slot_sem); + for (auto& worker : cc_workers) { + worker->stop(); + } + sem_destroy(&slot_sem); + printf("============ End %s ===========\n", test_name.c_str()); + } } -int sched_nr_sim_base::add_user(uint16_t rnti, - const sched_nr_interface::ue_cfg_t& ue_cfg_, - slot_point tti_rx, - uint32_t preamble_idx) +int sched_nr_base_tester::add_user(uint16_t rnti, + const sched_nr_interface::ue_cfg_t& ue_cfg_, + slot_point tti_rx, + uint32_t preamble_idx) { + sem_wait(&slot_sem); sched_ptr->ue_cfg(rnti, ue_cfg_); - std::lock_guard lock(std::mutex); TESTASSERT(ue_db.count(rnti) == 0); ue_db.insert(std::make_pair(rnti, sched_nr_ue_sim(rnti, ue_cfg_, current_slot_tx, preamble_idx))); @@ -136,11 +159,19 @@ int sched_nr_sim_base::add_user(uint16_t rnti, rach_info.msg3_size = 7; sched_ptr->dl_rach_info(ue_cfg_.carriers[0].cc, rach_info); + sem_post(&slot_sem); + return SRSRAN_SUCCESS; } -void sched_nr_sim_base::new_slot_(slot_point slot_tx) +void sched_nr_base_tester::run_slot(slot_point slot_tx) { + srsran_assert(not stopped.load(std::memory_order_relaxed), "Running scheduler when it has already been stopped"); + // Block concurrent or out-of-order calls to the scheduler + sem_wait(&slot_sem); + current_slot_tx = slot_tx; + nof_cc_remaining = cell_params.size(); + logger.set_context(slot_tx.to_uint()); mac_logger.set_context(slot_tx.to_uint()); @@ -157,20 +188,47 @@ void sched_nr_sim_base::new_slot_(slot_point slot_tx) set_external_slot_events(ue.second.get_ctxt(), events); apply_slot_events(ue.second.get_ctxt(), events); } + + slot_ctxt = get_enb_ctxt(); + slot_start_tp = std::chrono::steady_clock::now(); + + // Generate CC result (parallel or serialized) + uint32_t worker_idx = 0; + for (uint32_t cc = 0; cc < cell_params.size(); ++cc) { + if (worker_idx == cc_workers.size()) { + generate_cc_result(cc); + } else { + cc_workers[worker_idx]->push_task([this, cc]() { generate_cc_result(cc); }); + } + worker_idx = (worker_idx + 1) % (cc_workers.size() + 1); + } } -void sched_nr_sim_base::generate_cc_result_(uint32_t cc) +void sched_nr_base_tester::generate_cc_result(uint32_t cc) { // Run scheduler - auto tp1 = std::chrono::steady_clock::now(); sched_ptr->run_slot(current_slot_tx, cc, cc_results[cc].dl_res); sched_ptr->get_ul_sched(current_slot_tx, cc, cc_results[cc].ul_res); - auto tp2 = std::chrono::steady_clock::now(); - cc_results[cc].sched_latency_ns = std::chrono::duration_cast(tp2 - tp1); + auto tp2 = std::chrono::steady_clock::now(); + cc_results[cc].cc_latency_ns = std::chrono::duration_cast(tp2 - slot_start_tp); + + if (--nof_cc_remaining > 0) { + // there are still missing CC results + return; + } + + // Run tests and update UE state + process_results(); + + // Notify awaiting new slot worker + sem_post(&slot_sem); } -void sched_nr_sim_base::process_results() +void sched_nr_base_tester::process_results() { + // Derived class-defined tests + process_slot_result(slot_ctxt, cc_results); + sched_nr_cc_result_view cc_out; cc_out.slot = current_slot_tx; for (uint32_t cc = 0; cc < cell_params.size(); ++cc) { @@ -184,12 +242,7 @@ void sched_nr_sim_base::process_results() test_ssb_scheduled_grant(cc_out.slot, cell_params[cc_out.cc].cfg, cc_out.dl_cc_result->dl_sched.ssb); // Run UE-dedicated tests - sim_nr_enb_ctxt_t ctxt; - ctxt = get_enb_ctxt(); - test_dl_sched_result(ctxt, cc_out); - - // Derived class-defined tests - process_cc_result(cc_results[cc]); + test_dl_sched_result(slot_ctxt, cc_out); // Update UE state for (auto& u : ue_db) { @@ -198,7 +251,7 @@ void sched_nr_sim_base::process_results() } } -int sched_nr_sim_base::set_default_slot_events(const sim_nr_ue_ctxt_t& ue_ctxt, ue_nr_slot_events& pending_events) +int sched_nr_base_tester::set_default_slot_events(const sim_nr_ue_ctxt_t& ue_ctxt, ue_nr_slot_events& pending_events) { pending_events.cc_list.clear(); pending_events.cc_list.resize(cell_params.size()); @@ -229,7 +282,7 @@ int sched_nr_sim_base::set_default_slot_events(const sim_nr_ue_ctxt_t& ue_ctxt, return SRSRAN_SUCCESS; } -int sched_nr_sim_base::apply_slot_events(sim_nr_ue_ctxt_t& ue_ctxt, const ue_nr_slot_events& events) +int sched_nr_base_tester::apply_slot_events(sim_nr_ue_ctxt_t& ue_ctxt, const ue_nr_slot_events& events) { for (uint32_t enb_cc_idx = 0; enb_cc_idx < events.cc_list.size(); ++enb_cc_idx) { const auto& cc_feedback = events.cc_list[enb_cc_idx]; @@ -273,7 +326,7 @@ int sched_nr_sim_base::apply_slot_events(sim_nr_ue_ctxt_t& ue_ctxt, const ue_nr_ return SRSRAN_SUCCESS; } -sim_nr_enb_ctxt_t sched_nr_sim_base::get_enb_ctxt() const +sim_nr_enb_ctxt_t sched_nr_base_tester::get_enb_ctxt() const { sim_nr_enb_ctxt_t ctxt; ctxt.cell_params = cell_params; @@ -285,76 +338,4 @@ sim_nr_enb_ctxt_t sched_nr_sim_base::get_enb_ctxt() const return ctxt; } -} // namespace detail - -void sched_nr_sim::new_slot(slot_point slot_tx) -{ - current_slot_tx = slot_tx; - nof_cc_remaining = cell_params.size(); - this->new_slot_(slot_tx); -} - -void sched_nr_sim::generate_cc_result(uint32_t cc) -{ - // Run scheduler - this->generate_cc_result_(cc); - - if (--nof_cc_remaining > 0) { - // there are still missing CC results - return; - } - - // Run tests and update UE state - this->process_results(); -} - -void sched_nr_sim_parallel::new_slot(slot_point slot_tx) -{ - // Block concurrent or out-of-order calls to the scheduler - { - std::unique_lock lock(mutex); - while (nof_cc_remaining > 0 or (current_slot_tx.valid() and current_slot_tx + 1 != slot_tx)) { - cvar.wait(lock); - } - current_slot_tx = slot_tx; - nof_cc_remaining = cell_params.size(); - } - - // Run common new_slot updates - this->new_slot_(slot_tx); -} - -void sched_nr_sim_parallel::generate_cc_result(uint32_t cc) -{ - // Run scheduler - this->generate_cc_result_(cc); - - { - std::unique_lock lock(mutex); - if (--nof_cc_remaining > 0) { - // there are still missing CC results - return; - } - - // Run tests and update UE state - this->process_results(); - } - - // Notify waiting workers - cvar.notify_one(); -} - -sched_nr_sim_parallel::~sched_nr_sim_parallel() -{ - stop(); -} - -void sched_nr_sim_parallel::stop() -{ - std::unique_lock lock(mutex); - while (nof_cc_remaining > 0) { - cvar.wait(lock); - } -} - } // namespace srsenb \ No newline at end of file diff --git a/srsenb/test/mac/nr/sched_nr_sim_ue.h b/srsenb/test/mac/nr/sched_nr_sim_ue.h index 47f360a83..1c62b20e5 100644 --- a/srsenb/test/mac/nr/sched_nr_sim_ue.h +++ b/srsenb/test/mac/nr/sched_nr_sim_ue.h @@ -17,6 +17,11 @@ #include "srsenb/hdr/stack/mac/nr/sched_nr.h" #include "srsran/adt/circular_array.h" #include +#include + +namespace srsran { +class task_worker; +} namespace srsenb { @@ -98,10 +103,8 @@ private: sim_nr_ue_ctxt_t ctxt; }; -namespace detail { - /// Implementation of features common to sched_nr_sim_parallel and sched_nr_sim -class sched_nr_sim_base +class sched_nr_base_tester { public: struct cc_result_t { @@ -109,13 +112,17 @@ public: uint32_t cc; sched_nr_interface::dl_sched_res_t dl_res; sched_nr_interface::ul_sched_t ul_res; - std::chrono::nanoseconds sched_latency_ns; + std::chrono::nanoseconds cc_latency_ns; }; - sched_nr_sim_base(const sched_nr_interface::sched_args_t& sched_args, - const std::vector& cell_params_, - std::string test_name); - virtual ~sched_nr_sim_base(); + sched_nr_base_tester(const sched_nr_interface::sched_args_t& sched_args, + const std::vector& cell_params_, + std::string test_name, + uint32_t nof_workers = 1); + virtual ~sched_nr_base_tester(); + + void run_slot(slot_point slot_tx); + void stop(); int add_user(uint16_t rnti, const sched_nr_interface::ue_cfg_t& ue_cfg_, slot_point tti_rx, uint32_t preamble_idx); @@ -125,11 +132,10 @@ public: virtual void set_external_slot_events(const sim_nr_ue_ctxt_t& ue_ctxt, ue_nr_slot_events& pending_events) {} // configurable by simulator concrete implementation - virtual void process_cc_result(const cc_result_t& cc_out) {} + virtual void process_slot_result(const sim_nr_enb_ctxt_t& enb_ctxt, srsran::const_span cc_out) {} protected: - void new_slot_(slot_point slot_tx); - void generate_cc_result_(uint32_t cc); + void generate_cc_result(uint32_t cc); sim_nr_enb_ctxt_t get_enb_ctxt() const; int set_default_slot_events(const sim_nr_ue_ctxt_t& ue_ctxt, ue_nr_slot_events& pending_events); @@ -144,77 +150,19 @@ protected: std::unique_ptr sched_ptr; std::vector cell_params; - uint32_t nof_cc_remaining = 0; - slot_point current_slot_tx; - std::vector cc_results; + std::vector > cc_workers; std::map ue_db; -}; -} // namespace detail - -class sched_nr_sim_parallel : public detail::sched_nr_sim_base -{ - using base_t = detail::sched_nr_sim_base; - -public: - using base_t::base_t; - ~sched_nr_sim_parallel(); - - void stop(); - - int add_user(uint16_t rnti, const sched_nr_interface::ue_cfg_t& ue_cfg_, slot_point tti_rx, uint32_t preamble_idx) - { - std::lock_guard lock(mutex); - return base_t::add_user(rnti, ue_cfg_, tti_rx, preamble_idx); - } - slot_point get_slot_tx() const - { - std::lock_guard lock(mutex); - return current_slot_tx; - } - - void new_slot(slot_point slot_tx); - void generate_cc_result(uint32_t cc); - -private: - mutable std::mutex mutex; - std::condition_variable cvar; -}; - -class sched_nr_sim : public detail::sched_nr_sim_base -{ - using base_t = detail::sched_nr_sim_base; - -public: - using sched_nr_sim_base::sched_nr_sim_base; - - void new_slot(slot_point slot_tx); - void generate_cc_result(uint32_t cc); - - sched_nr_ue_sim& at(uint16_t rnti) { return ue_db.at(rnti); } - const sched_nr_ue_sim& at(uint16_t rnti) const { return ue_db.at(rnti); } - sched_nr_ue_sim* find_rnti(uint16_t rnti) - { - auto it = ue_db.find(rnti); - return it != ue_db.end() ? &it->second : nullptr; - } - const sched_nr_ue_sim* find_rnti(uint16_t rnti) const - { - auto it = ue_db.find(rnti); - return it != ue_db.end() ? &it->second : nullptr; - } - const sched_nr_interface::ue_cfg_t* get_user_cfg(uint16_t rnti) const - { - const sched_nr_ue_sim* ret = find_rnti(rnti); - return ret == nullptr ? nullptr : &ret->get_ctxt().ue_cfg; - } - bool user_exists(uint16_t rnti) const { return ue_db.count(rnti) > 0; } - sched_nr* get_sched() { return sched_ptr.get(); } - std::map::iterator begin() { return ue_db.begin(); } - std::map::iterator end() { return ue_db.end(); } + // slot-specific + slot_point current_slot_tx; + std::chrono::steady_clock::time_point slot_start_tp; + sim_nr_enb_ctxt_t slot_ctxt; + std::vector cc_results; - slot_point get_slot_tx() const { return current_slot_tx; } + std::atomic stopped{false}; + mutable sem_t slot_sem; + std::atomic nof_cc_remaining{0}; }; } // namespace srsenb diff --git a/srsenb/test/mac/nr/sched_nr_test.cc b/srsenb/test/mac/nr/sched_nr_test.cc index 1cd6f18b0..0f36b9c42 100644 --- a/srsenb/test/mac/nr/sched_nr_test.cc +++ b/srsenb/test/mac/nr/sched_nr_test.cc @@ -14,7 +14,6 @@ #include "sched_nr_sim_ue.h" #include "srsran/common/phy_cfg_nr_default.h" #include "srsran/common/test_common.h" -#include "srsran/common/thread_pool.h" #include namespace srsenb { @@ -24,38 +23,45 @@ using dl_sched_t = sched_nr_interface::dl_sched_t; static const srsran::phy_cfg_nr_t default_phy_cfg = srsran::phy_cfg_nr_default_t{srsran::phy_cfg_nr_default_t::reference_cfg_t{}}; -class sched_nr_tester : public sched_nr_sim_parallel +class sched_nr_tester : public sched_nr_base_tester { public: - using sched_nr_sim_parallel::sched_nr_sim_parallel; + using sched_nr_base_tester::sched_nr_base_tester; - void process_cc_result(const cc_result_t& cc_result) override + void process_slot_result(const sim_nr_enb_ctxt_t& slot_ctxt, srsran::const_span cc_list) override { - tot_latency_sched_ns += cc_result.sched_latency_ns.count(); - result_count++; - pdsch_count += cc_result.dl_res.dl_sched.pdcch_dl.size(); - - TESTASSERT(cc_result.dl_res.dl_sched.pdcch_dl.size() <= 1); - if (srsran_duplex_nr_is_dl(&cell_params[cc_result.cc].cfg.duplex, 0, current_slot_tx.slot_idx())) { - TESTASSERT(cc_result.dl_res.dl_sched.pdcch_dl.size() == 1 or not cc_result.dl_res.dl_sched.ssb.empty()); + tot_latency_sched_ns += + std::max_element(cc_list.begin(), cc_list.end(), [](const cc_result_t& lhs, const cc_result_t& rhs) { + return lhs.cc_latency_ns < rhs.cc_latency_ns; + })->cc_latency_ns.count(); + + for (auto& cc_out : cc_list) { + pdsch_count += cc_out.dl_res.dl_sched.pdcch_dl.size(); + cc_res_count++; + + TESTASSERT(cc_out.dl_res.dl_sched.pdcch_dl.size() <= 1); + if (srsran_duplex_nr_is_dl(&cell_params[cc_out.cc].cfg.duplex, 0, current_slot_tx.slot_idx())) { + TESTASSERT(cc_out.dl_res.dl_sched.pdcch_dl.size() == 1 or not cc_out.dl_res.dl_sched.ssb.empty()); + } } } void print_results() const { - test_logger.info("TESTER: %f PDSCH/{slot,cc} were allocated", pdsch_count / (double)result_count); + test_logger.info("TESTER: %f PDSCH/{slot,cc} were allocated", pdsch_count / (double)cc_res_count); srslog::flush(); } srslog::basic_logger& test_logger = srslog::fetch_basic_logger("TEST"); uint64_t tot_latency_sched_ns = 0; - uint32_t result_count = 0; + uint32_t cc_res_count = 0; uint32_t pdsch_count = 0; }; -void sched_nr_cfg_serialized_test() +void run_sched_nr_test(uint32_t nof_workers) { + srsran_assert(nof_workers > 0, "There must be at least one worker"); uint32_t max_nof_ttis = 1000, nof_sectors = 4; uint16_t rnti = 0x4601; @@ -64,7 +70,11 @@ void sched_nr_cfg_serialized_test() std::vector cells_cfg = get_default_cells_cfg(nof_sectors); - sched_nr_tester tester(cfg, cells_cfg, "Serialized Test"); + std::string test_name = "Serialized Test"; + if (nof_workers > 1) { + test_name = fmt::format("Parallel Test with {} workers", nof_workers); + } + sched_nr_tester tester(cfg, cells_cfg, test_name, nof_workers); sched_nr_interface::ue_cfg_t uecfg = get_default_ue_cfg(nof_sectors); tester.add_user(rnti, uecfg, slot_point{0, 0}, 0); @@ -72,10 +82,7 @@ void sched_nr_cfg_serialized_test() for (uint32_t nof_slots = 0; nof_slots < max_nof_ttis; ++nof_slots) { slot_point slot_rx(0, nof_slots % 10240); slot_point slot_tx = slot_rx + TX_ENB_DELAY; - tester.new_slot(slot_tx); - for (uint32_t cc = 0; cc != cells_cfg.size(); ++cc) { - tester.generate_cc_result(cc); - } + tester.run_slot(slot_tx); } tester.stop(); @@ -87,68 +94,21 @@ void sched_nr_cfg_serialized_test() printf("Total time taken per slot: %f usec\n", final_avg_usec); } -void sched_nr_cfg_parallel_cc_test() -{ - uint32_t nof_sectors = 4; - uint32_t max_nof_ttis = 1000; - uint16_t rnti = 0x4601; - - // Initiate CC Workers - std::vector > cc_workers; - cc_workers.reserve(nof_sectors - 1); - for (uint32_t i = 0; i < nof_sectors - 1; ++i) { - fmt::memory_buffer fmtbuf; - fmt::format_to(fmtbuf, "worker{}", i); - cc_workers.emplace_back(new srsran::task_worker{to_string(fmtbuf), 10}); - } - - sched_nr_interface::sched_args_t cfg; - cfg.auto_refill_buffer = true; - - std::vector cells_cfg = get_default_cells_cfg(nof_sectors); - - sched_nr_tester tester(cfg, cells_cfg, "Parallel CC Test"); - - sched_nr_interface::ue_cfg_t uecfg = get_default_ue_cfg(cells_cfg.size()); - tester.add_user(rnti, uecfg, slot_point{0, 0}, 0); - - for (uint32_t nof_slots = 0; nof_slots < max_nof_ttis; ++nof_slots) { - slot_point slot_rx(0, nof_slots % 10240); - slot_point slot_tx = slot_rx + TX_ENB_DELAY; - tester.new_slot(slot_tx); - // Run scheduler in parallel for {0, cc-2} and in the same thread for last cc - for (uint32_t cc = 0; cc != cells_cfg.size() - 1; ++cc) { - cc_workers[cc]->push_task([cc, &tester]() { tester.generate_cc_result(cc); }); - } - tester.generate_cc_result(cells_cfg.size() - 1); - } - - // Wait for all jobs to finish - tester.stop(); - - tester.print_results(); - - // TESTASSERT(tasks.pdsch_count == (int)(max_nof_ttis * nof_sectors * 0.6)); - - double final_avg_usec = tester.tot_latency_sched_ns; - final_avg_usec = final_avg_usec / 1000.0 / max_nof_ttis / nof_sectors; - printf("Total time taken per slot [usec]: %f\n", final_avg_usec); -} - } // namespace srsenb int main() { auto& test_logger = srslog::fetch_basic_logger("TEST"); - test_logger.set_level(srslog::basic_levels::error); + test_logger.set_level(srslog::basic_levels::warning); auto& mac_nr_logger = srslog::fetch_basic_logger("MAC-NR"); - mac_nr_logger.set_level(srslog::basic_levels::error); + mac_nr_logger.set_level(srslog::basic_levels::warning); auto& pool_logger = srslog::fetch_basic_logger("POOL"); pool_logger.set_level(srslog::basic_levels::debug); // Start the log backend. srslog::init(); - srsenb::sched_nr_cfg_serialized_test(); - srsenb::sched_nr_cfg_parallel_cc_test(); + srsenb::run_sched_nr_test(1); + srsenb::run_sched_nr_test(2); + srsenb::run_sched_nr_test(4); }