diff --git a/.gitmodules b/.gitmodules index 478789e..d37f04a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,9 +13,6 @@ [submodule "third_party/glob"] path = third_party/glob url = https://github.com/p-ranav/glob.git -[submodule "third_party/matplotplusplus"] - path = third_party/matplotplusplus - url = https://github.com/alandefreitas/matplotplusplus.git [submodule "third_party/pybind11"] path = third_party/pybind11 url = https://github.com/pybind/pybind11.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b5de2f..30f76b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,6 @@ include(${oops_3rd_cmake_dir}/fmt.cmake) include(${oops_3rd_cmake_dir}/scnlib.cmake) include(${oops_3rd_cmake_dir}/glob.cmake) include(${oops_3rd_cmake_dir}/pybind11.cmake) -include(${oops_3rd_cmake_dir}/matplotplusplus.cmake) if(ENABLE_TEST) include(${oops_3rd_cmake_dir}/googletest.cmake) endif() diff --git a/module/common/include/oops/str.h b/module/common/include/oops/str.h index 07a3d0f..e616f23 100644 --- a/module/common/include/oops/str.h +++ b/module/common/include/oops/str.h @@ -39,6 +39,11 @@ void Split(std::string_view s, const char *delim, Iter iter) { Split(s, delim, false, iter); } +template +void Split(std::string_view s, char delim, Iter iter) { + Split(s, std::string_view(&delim, 1), false, iter); +} + template void SplitMultiDelim(std::string_view s, std::string_view delims, bool skip_empty, Iter iter) { std::size_t begin{0}; @@ -167,6 +172,8 @@ constexpr std::string_view Strip(std::string_view s, std::string_view chars = SP return s.substr(pos, s.find_last_not_of(chars) - pos + 1); } +std::string Elide(std::string_view s, std::size_t n); + template ::std::string ToStr(const T &t) { ::std::ostringstream oss; diff --git a/module/common/src/str.cpp b/module/common/src/str.cpp index 409266f..98b94ee 100644 --- a/module/common/src/str.cpp +++ b/module/common/src/str.cpp @@ -28,4 +28,14 @@ std::string ToLower(std::string_view s) noexcept { } return res; } + +std::string Elide(std::string_view s, std::size_t n) { + if (s.size() <= n) { + return std::string{s}; + } + if (n <= 3) { + return std::string(n, '.'); + } + return std::string{s.substr(0, n - 3)} + "..."; +} } // namespace oops diff --git a/module/profiling/tool/monitor/CMakeLists.txt b/module/profiling/tool/monitor/CMakeLists.txt index d7df0eb..b04a267 100644 --- a/module/profiling/tool/monitor/CMakeLists.txt +++ b/module/profiling/tool/monitor/CMakeLists.txt @@ -2,4 +2,4 @@ file(GLOB_RECURSE SRC "*.cpp") add_executable(oops_profiling_monitor ${SRC}) set_target_properties(oops_profiling_monitor PROPERTIES OUTPUT_NAME monitor) -target_link_libraries(oops_profiling_monitor PRIVATE pthread argparse Glob oops_profiling_s) +target_link_libraries(oops_profiling_monitor PRIVATE pthread argparse Glob pybind11::embed oops_profiling_s) diff --git a/module/profiling/tool/monitor/monitor.cpp b/module/profiling/tool/monitor/monitor.cpp index ec9fc3f..1864047 100644 --- a/module/profiling/tool/monitor/monitor.cpp +++ b/module/profiling/tool/monitor/monitor.cpp @@ -18,6 +18,8 @@ #include "argparse/argparse.hpp" #include "glob/glob.h" +#include "pybind11/embed.h" +#include "pybind11/stl.h" #include "oops/cpu_timer.h" #include "oops/enum_bitset.h" @@ -26,6 +28,7 @@ #include "oops/unit.h" namespace fs = std::filesystem; +namespace py = pybind11; // 配置全局变量 enum class ProgramType : uint8_t { MEASURE, REPORT }; @@ -56,13 +59,20 @@ struct MetricEntry { std::size_t width{6}; MetricGroup group{}; }; -inline MetricEntry METRIC_TABLE[] = { - {"Cpu(%)", 6, MetricGroup::CPU}, - {"Rss(G)", 6, MetricGroup::MEMORY}, - {"Hwm(G)", 6, MetricGroup::MEMORY}, - {"Swap(G)", 6, MetricGroup::MEMORY}}; -// 公共函数 +constexpr MetricEntry METRIC_TABLE[] = { + {"%Cpu", 6, MetricGroup::CPU}, + {"Rss(G)", 8, MetricGroup::MEMORY}, + {"Hwm(G)", 8, MetricGroup::MEMORY}, + {"Swap(G)", 8, MetricGroup::MEMORY}}; + +// 测量功能 +bool ProcExist(pid_t pid) { + static std::string path{"/proc/" + std::to_string(pid)}; + struct stat buffer; + return stat(path.c_str(), &buffer) == 0; +} + std::string TimestampToStr(std::chrono::system_clock::time_point time_point) { std::time_t time{std::chrono::system_clock::to_time_t(time_point)}; std::tm *tm{std::localtime(&time)}; // std::localtime不可重入 @@ -75,145 +85,6 @@ std::string TimestampToStr(std::chrono::system_clock::time_point time_point) { return oss.str(); } -std::chrono::system_clock::time_point StrToTimestamp(std::string_view sv) { - if (sv.size() < 26) { - throw std::runtime_error("size not match"); - } - std::tm tm{}; - - // 提取"YYYY-mm-dd HH:MM:SS" - std::istringstream iss{std::string(sv.substr(0, 19))}; - iss >> std::get_time(&tm, "%Y-%m-%d %H:%M:%S"); - auto time_point{std::chrono::system_clock::from_time_t(std::mktime(&tm))}; - - // 解析微秒 - std::size_t us; - std::from_chars(sv.data() + 20, sv.data() + 26, us); - return time_point + std::chrono::microseconds(us); -} - -bool ProcExist(pid_t pid) { - static std::string path{"/proc/" + std::to_string(pid)}; - struct stat buffer; - return stat(path.c_str(), &buffer) == 0; -} - -struct SampleEntry { - SampleEntry(std::string s) : name{std::move(s)} {} - SampleEntry(std::string_view s) : name{s} {} - std::string name; - std::vector values; -}; - -struct RawDataEntry { - fs::path raw_file_path; - pid_t pid{}; - std::size_t interval{}; - std::chrono::system_clock::time_point start_time; - std::vector sample_table; -}; - -inline std::vector RAW_DATA_TABLE; - -struct ParseValueAfterPrefixResult { - explicit operator bool() const noexcept { return matched && parsed; } - std::string_view remain; - bool matched{false}; - bool parsed{false}; -}; - -template -auto ParseValueAfterPrefix(std::string_view s, std::string_view prefix, T &value) noexcept { - ParseValueAfterPrefixResult res; - s = oops::Strip(s); - - auto pos{s.find(prefix)}; - if (pos == std::string_view::npos) { - return res; - } - s.remove_prefix(pos + prefix.size()); - s = oops::Strip(s); // 使用StripLeft优化 - res.matched = true; - res.remain = s; - - auto [ptr, errc]{std::from_chars(s.data(), s.data() + s.size(), value)}; - if (errc != std::errc{}) { - return res; - } - s.remove_prefix(ptr - s.data()); - res.parsed = true; - res.remain = oops::Strip(s); - return res; -} - -void ParseRawData(std::istream &is, RawDataEntry &entry) { - using namespace std::literals; - - // 解析阶段控制变量 - enum class Step { META = 0, HEADER_ROW, DATA_ROW } step{}; - std::size_t metric_num{}; - - std::string buf; - while (std::getline(is, buf)) { - std::string_view line{buf}; - line = oops::Strip(line); - if (line.empty()) { - continue; - } - - switch (step) { - case Step::META: { - if (ParseValueAfterPrefix(line, "Tracked pid:"sv, entry.pid)) { - break; - } - if (ParseValueAfterPrefix(line, "Interval:"sv, entry.interval)) { - break; - } - std::size_t pos{line.find("Timestamp:")}; // 提取解析函数优化 - if (pos != std::string_view::npos) { - entry.start_time = StrToTimestamp(line.substr(pos + 10)); - break; - } - if (line.find("Monitoring results:") != std::string_view::npos) { - step = Step::HEADER_ROW; - } - break; - } - - case Step::HEADER_ROW: { - auto tokens{oops::Split>(line)}; - metric_num = tokens.size(); - for (auto token : tokens) { - entry.sample_table.emplace_back(token); - } - step = Step::DATA_ROW; - break; - } - - case Step::DATA_ROW: { - auto tokens{oops::Split>(line)}; - if (metric_num == 0) { - throw std::runtime_error("bad metric num"); - } - if (metric_num != tokens.size()) { - throw std::runtime_error("bad metric num"); - } - for (std::size_t i{0}; i < tokens.size(); ++i) { - double d{}; - std::from_chars(tokens[i].data(), tokens[i].data() + tokens[i].size(), d); - entry.sample_table[i].values.push_back(d); - } - break; - } - } - } -} - -void ParseRawData(const fs::path &path, RawDataEntry &entry) { - std::ifstream ifs(path); - ParseRawData(ifs, entry); -} - std::string MakeHeaderRow() { std::ostringstream oss; oss << std::fixed << std::setprecision(2); @@ -248,6 +119,7 @@ std::string MakeDataRow(const std::vector &values) { void Measure() { std::cout << std::fixed << std::setprecision(2); + std::cout << "Pid: " << getpid() << std::endl; std::cout << "Tracked pid: " << ARGS.measure.pid << std::endl; std::cout << "Interval: " << ARGS.measure.itv << "s" << std::endl; std::cout << "Ticks per sec: " << oops::GetTicksPerSec() << std::endl; @@ -272,7 +144,7 @@ void Measure() { // 根据选项配置完成测量 if (ENABLED_METRIC_GROUP.Test(MetricGroup::CPU)) { // 有限测量区间指标，再测量单点指标 - values[oops::ToUnderlying(Metrics::CPU_USAGE)] = cpu_timer.Lap().CpuUsage(); + values[oops::ToUnderlying(Metrics::CPU_USAGE)] = 100 * cpu_timer.Lap().CpuUsage(); } if (ENABLED_METRIC_GROUP.Test(MetricGroup::MEMORY)) { @@ -291,6 +163,44 @@ void Measure() { } } +// 报告全局变量 +struct MetricDataEntry { + MetricDataEntry(std::string s) : name{std::move(s)} {} + MetricDataEntry(std::string_view s) : name{s} {} + std::string name; + std::vector values; +}; + +struct RawDataEntry { + RawDataEntry(std::string s) : raw_file_name{std::move(s)} {} + std::string raw_file_name; + pid_t pid{}; + double interval{}; + std::chrono::system_clock::time_point start_time; + std::vector metric_data_table; +}; + +inline std::vector RAW_DATA_TABLE; + +// 报告功能 +std::chrono::system_clock::time_point StrToTimestamp(std::string_view sv) { + sv = oops::Strip(sv); + if (sv.size() < 26) { + throw std::runtime_error("size too small for timestamp format"); + } + std::tm tm{}; + + // 提取"YYYY-mm-dd HH:MM:SS" + std::istringstream iss{std::string(sv.substr(0, 19))}; + iss >> std::get_time(&tm, "%Y-%m-%d %H:%M:%S"); + auto time_point{std::chrono::system_clock::from_time_t(std::mktime(&tm))}; + + // 解析微秒 + std::size_t us{}; + std::from_chars(sv.data() + 20, sv.data() + 26, us); + return time_point + std::chrono::microseconds(us); +} + void SearchValidInput() { std::vector literal_input; std::vector patterns; @@ -311,6 +221,7 @@ void SearchValidInput() { if (input.find_first_of("*?") != std::string::npos) { // glob相对路径，那么在dir中递归查找glob patterns.push_back((dir / "**" / input_path).string()); + patterns.push_back((dir / input_path).string()); } else { // 普通相对路径，仅拼接 literal_input.push_back(dir / input_path); @@ -348,13 +259,205 @@ void SearchValidInput() { ARGS.report.valid_input = std::move(valid_input); } +template +auto ParseValueAfterPrefix(std::string_view s, std::string_view prefix, T &value) noexcept { + s = oops::Strip(s); + struct { + explicit operator bool() const noexcept { return matched && parsed; } + std::string_view remain; + bool matched{false}; + bool parsed{false}; + } res; + + auto pos{s.find(prefix)}; + if (pos == std::string_view::npos) { + return res; + } + s.remove_prefix(pos + prefix.size()); + s = oops::Strip(s); // 使用StripLeft优化 + res.matched = true; + res.remain = s; + + auto [ptr, errc]{std::from_chars(s.data(), s.data() + s.size(), value)}; + if (errc != std::errc{}) { + return res; + } + s.remove_prefix(ptr - s.data()); + res.parsed = true; + res.remain = oops::Strip(s); + return res; +} + +void ParseRawData(std::istream &is, RawDataEntry &entry) { + using namespace std::literals; + + // 解析阶段控制变量 + enum class Step { META = 0, HEADER_ROW, DATA_ROW } step{}; + std::size_t metric_num{}; + + std::string buf; + while (std::getline(is, buf)) { + std::string_view line{buf}; + line = oops::Strip(line); + if (line.empty()) { + continue; + } + + switch (step) { + case Step::META: { + if (ParseValueAfterPrefix(line, "Tracked pid:"sv, entry.pid)) { + break; + } + if (ParseValueAfterPrefix(line, "Interval:"sv, entry.interval)) { + break; + } + std::size_t pos{line.find("Timestamp:")}; // 提取解析函数优化 + if (pos != std::string_view::npos) { + entry.start_time = StrToTimestamp(line.substr(pos + 10)); + break; + } + if (line.find("Monitoring results:") != std::string_view::npos) { + step = Step::HEADER_ROW; + } + break; + } + + case Step::HEADER_ROW: { + auto tokens{oops::Split>(line, ", ")}; + metric_num = tokens.size(); + for (auto token : tokens) { + entry.metric_data_table.emplace_back(oops::Strip(token)); + } + step = Step::DATA_ROW; + break; + } + + case Step::DATA_ROW: { + auto tokens{oops::Split>(line, ", ")}; + if (metric_num == 0) { + throw std::runtime_error("bad metric num"); + } + if (metric_num != tokens.size()) { + throw std::runtime_error("bad metric num"); + } + for (std::size_t i{0}; i < tokens.size(); ++i) { + double d{}; + auto token{oops::Strip(tokens[i])}; + std::from_chars(token.data(), token.data() + token.size(), d); + entry.metric_data_table[i].values.push_back(d); + } + break; + } + } + assert(step == Step::DATA_ROW); + } +} + +void ParseRawData(const fs::path &path, RawDataEntry &entry) { + std::ifstream ifs(path); + ParseRawData(ifs, entry); +} + +void PlotCpu() { + if (RAW_DATA_TABLE.empty()) { + return; + } + + // 识别记录了CPU利用率的raw data idx及对应CPU利用率的metric data dix + std::vector> valid_idxs; + for (std::size_t i{0}; i < RAW_DATA_TABLE.size(); ++i) { + const auto &r_entry{RAW_DATA_TABLE[i]}; + for (std::size_t j{0}; j < r_entry.metric_data_table.size(); ++j) { + const auto &m_entry{r_entry.metric_data_table.at(j)}; + if (m_entry.name == METRIC_TABLE[oops::ToUnderlying(Metrics::CPU_USAGE)].name && !m_entry.values.empty()) { + valid_idxs.emplace_back(i, j); + } + } + } + if (valid_idxs.empty()) { + return; + } + + // 最早start_time + std::chrono::system_clock::time_point start_time{std::chrono::system_clock::time_point::max()}; + for (const auto &p : valid_idxs) { + start_time = std::min(start_time, RAW_DATA_TABLE[p.first].start_time); + } + + // 最大数值 + double max_value{std::numeric_limits::min()}; + for (const auto &[r_idx, m_idx] : valid_idxs) { + for (double value : RAW_DATA_TABLE[r_idx].metric_data_table[m_idx].values) { + max_value = std::max(max_value, value); + } + } + + try { + py::scoped_interpreter guard{}; // 绘图函数仅执行单次 + auto plt{py::module_::import("matplotlib.pyplot")}; + auto subplots{plt.attr("subplots")( + valid_idxs.size(), 1, py::arg("sharex") = true, + py::arg("figsize") = py::make_tuple( + 16, std::max(std::min(9, 3 * valid_idxs.size()), valid_idxs.size())))}; + + py::list axes; // 存储所有子图，使用vector反而会有拷贝和引用计数开销 + if (valid_idxs.size() == 1) { + axes.append(subplots[py::int_(1)]); + } else { + axes = subplots[py::int_(1)].cast(); + } + + for (std::size_t i{0}; i < valid_idxs.size(); ++i) { + const auto &r_entry{RAW_DATA_TABLE[valid_idxs[i].first]}; + const auto &m_entry{r_entry.metric_data_table[valid_idxs[i].second]}; + double offset{std::chrono::duration{r_entry.start_time - start_time}.count()}; + + // 构造时间数组，由于offset差异，每个raw data需要单独构造 + std::vector times; + for (std::size_t j{0}; j < m_entry.values.size(); ++j) { + times.push_back(offset + j * r_entry.interval); + } + + auto ax{axes[i]}; + // 使用step绘制阶梯图，post逻辑：第0个点，时刻是0，对应[0, itv]区间的平均CPU利用率 + ax.attr("step")( + times, m_entry.values, py::arg("where") = "post", py::arg("label") = m_entry.name, + py::arg("linewidth") = 1.5); + ax.attr("margins")(py::arg("x") = 0); + ax.attr("grid")(true, py::arg("linestyle") = "--", py::arg("alpha") = 0.5); + double ylim{std::max(100., std::ceil(max_value / 100) * 100)}; + ax.attr("set_ylim")(-ylim / 20, ylim + ylim / 20); // 对齐每个子图y坐标范围 + // 绘制每个子图左侧描述raw data的标签 + ax.attr("set_ylabel")( + "#" + std::to_string(i) + "\nPid: " + std::to_string(r_entry.pid) + "\n" + + oops::Elide(r_entry.raw_file_name, 10), + py::arg("rotation") = 0, py::arg("labelpad") = 30, py::arg("va") = "center"); + ax.attr("legend")(py::arg("loc") = "upper right"); + } + + plt.attr("suptitle")( + "CPU Utilization Time Series (%)", py::arg("fontsize") = 16, py::arg("fontweight") = "bold"); + plt.attr("xlabel")("Time (s)"); + plt.attr("tight_layout")(); + plt.attr("show")(); + } catch (const py::error_already_set &e) { + std::cerr << "Error: " << e.what() << std::endl; + exit(1); + } +} + void Report() { + // 搜索日志文件 SearchValidInput(); - std::size_t raw_file_num{ARGS.report.valid_input.size()}; - RAW_DATA_TABLE.resize(raw_file_num); - for (std::size_t i{0}; i < raw_file_num; ++i) { - ParseRawData(ARGS.report.valid_input[i], RAW_DATA_TABLE[i]); + + // 解析数据 + for (const auto &p : ARGS.report.valid_input) { + RAW_DATA_TABLE.emplace_back(p.filename().string()); + ParseRawData(p, RAW_DATA_TABLE.back()); } + + // 绘制CPU利用率折线图 + PlotCpu(); } // 参数解析 @@ -377,14 +480,20 @@ void ParseArgs(int argc, char *argv[]) { report.add_argument("input").help("comma-separated sequence of raw data file").required(); report.add_argument("-d", "--dir") .help("the root directory to search for files matching the 'input' pattern") - .default_value(std::filesystem::current_path().string()); + .default_value("."); program.add_subparser(report); try { program.parse_args(argc, argv); } catch (const std::exception &e) { std::cerr << "Error: " << e.what() << std::endl; - std::cerr << program; + if (program.is_subcommand_used(measure)) { + std::cerr << measure; + } else if (program.is_subcommand_used(report)) { + std::cerr << report; + } else { + std::cerr << program; + } exit(1); } @@ -452,8 +561,6 @@ void ParseArgs(int argc, char *argv[]) { << std::endl; ARGS.report.dir.emplace_back(fs::current_path()); } - } else { - std::cout << program << std::endl; } } diff --git a/third_party/cmake/matplotplusplus.cmake b/third_party/cmake/matplotplusplus.cmake deleted file mode 100644 index e8e5894..0000000 --- a/third_party/cmake/matplotplusplus.cmake +++ /dev/null @@ -1,13 +0,0 @@ -# matplotplusplus-1.2.2 (fe308e13) -set(lib_dir ${oops_3rd_dir}/matplotplusplus) -if(NOT EXISTS "${lib_dir}/CMakeLists.txt") - execute_process(COMMAND git submodule update --init --recursive ${lib_dir}) -endif() - -add_subdirectory(${lib_dir}) -if(TARGET nodesoup) - target_compile_options(nodesoup PRIVATE -w) -endif() -if(TARGET matplot) - target_compile_options(matplot PRIVATE -w) -endif() diff --git a/third_party/matplotplusplus b/third_party/matplotplusplus deleted file mode 160000 index fe308e1..0000000 --- a/third_party/matplotplusplus +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fe308e13f6c3ac7c5bd2b2f4717961d1d56d07ea