peranger/AimRT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

run all examples of aimrt and generate the report (#34)

Browse Source 
* run all examples of aimrt and generate the report

* add example items  of cpp and python

* add release_notes

* Modify copyright statement

---------

Co-authored-by: hanjun <hanjun@agibot.com>
This commit is contained in:
han J 2024-10-18 16:41:25 +08:00 committed by GitHub
parent 1186d81a07
commit d1ea3b4e13
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 1158 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
document/sphinx-cn/release_notes/v0_9_0.md
View File

@ -19,3 +19,5 @@
- 新增了 aimrt_py rpc benchmark 示例；
- iceoryx 插件在编译前先检查是否存在libacl，不存在则不进行编译；
- 提供 RPC 服务的插件现在支持指定 service name；
- 提供一键运行example的脚本，并生成测试报告；

103
src/examples/utils/common.py Normal file
View File

@ -0,0 +1,103 @@
# Copyright (c) 2023, AgiBot Inc.
# All rights reserved.
import os
# ANSI escape codes for coloring the output
RESET = "\033[0m"
BOLD = "\033[1m"
RED = "\033[31m"
GREEN = "\033[32m"
YELLOW = "\033[33m"
BLUE = "\033[34m"
MAGENTA = "\033[35m"
CYAN = "\033[36m"
WHITE = "\033[37m"
BRIGHT_GREEN = "\033[92m"
# Test result codes
class TestResult:
SUCCESS = 0
EXPECTED_OUTPUT_NOT_FOUND = 1
FORBIDDEN_OUTPUT_FOUND = 2
EXIT_STRING_NOT_FOUND = 3
# Common forbiden outputs for all modules
default_forbidden_outputs = [
"[Error]",
"[Fatal]",
"Segmentation fault" "core dumped",
"Traceback (most recent call last):",
]
# Default expected outputs for pb_chn_cpp
default_pb_chn_sub_expected_outputs_cpp = ['data: {"msg":"count: 1","num":1}']
default_pb_chn_pub_expected_outputs_cpp = ['Publish new pb event, data: {"msg":"count: 1","num":1}']
# Default expected outputs for pb_rpc_cpp
default_pb_rpc_srv_expected_outputs_cpp = [
'req: {"msg":"hello world foo, count 1"}, return rsp: {"code":"0","msg":"echo hello world foo, count 1"}'
]
default_pb_rpc_cli_expected_outputs_cpp = [
'req: {"msg":"hello world foo, count 1"}',
'rsp: {"code":"0","msg":"echo hello world foo, count 1"}',
]
# Default expected outputs for ros2_chn_cpp
default_ros2_chn_sub_expected_outputs_cpp = ["Receive new ros event, data:", "test_msg2"]
default_ros2_chn_pub_expected_outputs_cpp = ["Publish new ros event, data:", "test_msg2:"]
# Default expected outputs for ros2_rpc_cpp
default_ros2_rpc_srv_expected_outputs_cpp = ["Get new rpc call. context: Server context", "Svr rpc time cost"]
default_ros2_rpc_cli_expected_outputs_cpp = [
"start new rpc call. req:",
"Get rpc ret, status: suc, code 0, msg: OK. rsp:",
]
# Default expected outputs for pb_chn_python
default_pb_chn_sub_expected_outputs_python = ["Get new pb event, data: {", '"msg": "example msg"', '"num": 123456']
default_pb_chn_pub_expected_outputs_python = ["Publish new pb event, data: {", '"msg": "example msg"', '"num": 123456']
# Default expected outputs for pb_rpc_python
default_pb_rpc_srv_expected_outputs_python = ["Server handle new rpc call."]
default_pb_rpc_cli_expected_outputs_python = ["Call rpc done, status: suc, code 0, msg: OK"]
# Default exit string
default_exit_string = "AimRT exit."
# Default timeout, unit: second
default_timeout = 4
# Todo this is not the best way to find the aim directory, need to improve it
def upwards_find_aim_directory(aim: str = "build", start_directory: str = os.getcwd()) -> str:
current_directory = start_directory
# try to find the aim directory in the current directory
while True:
aim_directory = os.path.join(current_directory, aim)
if os.path.isdir(aim_directory):
return aim_directory
# move to the parent directory
parent_directory = os.path.dirname(current_directory)
# if come to the root directory, stop searching
if parent_directory == current_directory:
break
current_directory = parent_directory
assert False, f"Cannot find {aim} directory."
# Default build path
defualt_build_path = upwards_find_aim_directory()
# Default py_path
py_cwd = defualt_build_path + "/../src/examples/py"
default_save_path = os.path.join(os.getcwd(), "test_log")

696
src/examples/utils/example_items.py Normal file
View File

@ -0,0 +1,696 @@
# Copyright (c) 2023, AgiBot Inc.
# All rights reserved.
# this is a file to store the items for the example
# fotmat:
# ·script_path* # Required: Specifies the path to the script that will be executed.
# ·expected_outputs* # Required: The expected output result of the test.
# ·tags # recommended: Tags for categorizing the test item, facilitating management and retrieval.
# ·forbidden_outputs # Optional: Specifies output results that are not allowed.
# ·exit_string # Optional: Specifies the string that the script should output to indicate successful execution.
# ·timeout # Optional: Sets a timeout duration for the test execution to prevent it from hanging indefinitely.
# ·cwd # Optional: Specifies the current working directory, used as context when running the script.
# ·limits # Optional: Sets restrictions for executing the script, such as memory or time limit.
# ·priority # Optional: Specifies the priority of the test item for better organization during test execution.
# ·dependencies: # Optional: Lists other components or libraries that the test item depends on.
# ·description: # Optional: A detailed description of the test item, explaining its purpose and background.
# ·expansion: # Optional: Indicates any planned or potential expansion related to the test item.
from utils.common import *
# sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
test_items = [
# //////////////////////////////////////////////////////////////////////////////////////////
# ////////////////////////////////CPP EXAMPLES//////////////////////////////////////////////
# //////////////////////////////////////////////////////////////////////////////////////////
# ------------------------------iceoryx_pb_chn---------------------------------------------
{
"script_path": [
"./iox-roudi",
"./start_examples_plugins_iceoryx_plugin_pb_chn_sub.sh",
"./start_examples_plugins_iceoryx_plugin_pb_chn_pub.sh",
],
"expected_outputs": [
["ready"],
default_pb_chn_sub_expected_outputs_cpp,
default_pb_chn_pub_expected_outputs_cpp,
],
"exit_string": [
"",
default_exit_string,
default_exit_string,
],
"tags": ["all", "cpp", "plugins", "iceoryx", "pb", "chn"],
"limit": "iox-roudi",
},
# ------------------------------iceoryx_ros2_chn---------------------------------------------
{
"script_path": [
"./iox-roudi",
"./start_examples_plugins_iceoryx_plugin_ros2_chn_sub.sh",
"./start_examples_plugins_iceoryx_plugin_ros2_chn_pub.sh",
],
"expected_outputs": [
["ready"],
default_ros2_chn_sub_expected_outputs_cpp,
default_ros2_chn_pub_expected_outputs_cpp,
],
"exit_string": [
"",
default_exit_string,
default_exit_string,
],
"tags": ["all", "cpp", "plugins", "iceoryx", "ros2", "chn"],
"limit": "iox-roudi",
},
# -------------------------------log_control_plugin----------------------------------------
{
"script_path": [
"./start_examples_plugins_log_control_plugin.sh",
"./tools/log_control_plugin_get_lvl.sh",
],
"expected_outputs": [
["s = abc, n = 1"],
['{"code":0,"msg":"","module_log_level_map":'],
],
"forbidden_outputs": [
"",
default_forbidden_outputs,
],
"exit_string": [
default_exit_string,
"",
],
"tags": ["all", "cpp", "plugins", "log_control", "curl"],
"limit": "port:50080",
},
# -------------------------------------tcp_pb_chn----------------------------------------------
{
"script_path": [
"./start_examples_plugins_net_plugin_pb_chn_tcp_sub.sh",
"./start_examples_plugins_net_plugin_pb_chn_tcp_pub.sh",
],
"expected_outputs": [
default_pb_chn_sub_expected_outputs_cpp,
default_pb_chn_pub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "net", "tcp", "pb", "chn"],
},
# -------------------------------------udp_pb_chn----------------------------------------------
{
"script_path": [
"./start_examples_plugins_net_plugin_pb_chn_udp_sub.sh",
"./start_examples_plugins_net_plugin_pb_chn_udp_pub.sh",
],
"expected_outputs": [
default_pb_chn_sub_expected_outputs_cpp,
default_pb_chn_pub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "net", "udp", "pb", "chn"],
},
# -------------------------------------http_pb_chn----------------------------------------------
{
"script_path": [
"./start_examples_plugins_net_plugin_pb_chn_http_sub.sh",
"./start_examples_plugins_net_plugin_pb_chn_http_pub.sh",
],
"expected_outputs": [
default_pb_chn_sub_expected_outputs_cpp,
default_pb_chn_pub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "net", "http", "pb", "chn"],
"limit": "port:50080",
},
# -------------------------------------http_ros2_chn----------------------------------------------
{
"script_path": [
"./start_examples_plugins_net_plugin_ros2_chn_http_sub.sh",
"./start_examples_plugins_net_plugin_ros2_chn_http_pub.sh",
],
"expected_outputs": [
default_ros2_chn_sub_expected_outputs_cpp,
default_ros2_chn_pub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "net", "http", "ros2", "chn"],
"limit": "port:50080",
},
# -------------------------------------http_pb_rpc----------------------------------------------
{
"script_path": [
"./start_examples_plugins_net_plugin_pb_rpc_http_server.sh",
"./start_examples_plugins_net_plugin_pb_rpc_http_client.sh",
],
"expected_outputs": [
default_pb_rpc_srv_expected_outputs_cpp,
default_pb_rpc_cli_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "net", "http", "pb", "rpc"],
"limit": "port:50080",
},
# -------------------------------------http_ros2_rpc----------------------------------------------
{
"script_path": [
"./start_examples_plugins_net_plugin_ros2_rpc_http_server.sh",
"./start_examples_plugins_net_plugin_ros2_rpc_http_client.sh",
],
"expected_outputs": [
default_ros2_rpc_srv_expected_outputs_cpp,
default_ros2_rpc_cli_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "net", "http", "ros2", "rpc"],
"limit": "port:50080",
},
# ---------------------------------------zenoh_pb_chn--------------------------------------------
{
"script_path": [
"./start_examples_plugins_zenoh_plugin_pb_chn_sub.sh",
"./start_examples_plugins_zenoh_plugin_pb_chn_pub.sh",
],
"expected_outputs": [
default_pb_chn_sub_expected_outputs_cpp,
default_pb_chn_pub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "zenoh", "pb", "chn"],
},
# ---------------------------------------zenoh_ros2_chn------------------------------------------
{
"script_path": [
"./start_examples_plugins_zenoh_plugin_ros2_chn_sub.sh",
"./start_examples_plugins_zenoh_plugin_ros2_chn_pub.sh",
],
"expected_outputs": [
default_ros2_chn_sub_expected_outputs_cpp,
default_ros2_chn_pub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "zenoh", "ros2", "chn"],
},
# ----------------------------------------zenoh_pb_rpc-------------------------------------
{
"script_path": [
"./start_examples_plugins_zenoh_plugin_pb_rpc_server.sh",
"./start_examples_plugins_zenoh_plugin_pb_rpc_client.sh",
],
"expected_outputs": [
default_pb_rpc_srv_expected_outputs_cpp,
default_pb_rpc_cli_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "zenoh", "pb", "rpc"],
},
# -----------------------------------------zenoh_ros2_rpc------------------------------------
{
"script_path": [
"./start_examples_plugins_zenoh_plugin_ros2_rpc_server.sh",
"./start_examples_plugins_zenoh_plugin_ros2_rpc_client.sh",
],
"expected_outputs": [
default_ros2_rpc_srv_expected_outputs_cpp,
default_ros2_rpc_cli_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "zenoh", "ros2", "rpc"],
},
# ------------------------------------------mqtt_pb_chn-----------------------------------
{
"script_path": [
"./start_examples_plugins_mqtt_plugin_pb_chn_sub.sh",
"./start_examples_plugins_mqtt_plugin_pb_chn_pub.sh",
],
"expected_outputs": [
default_pb_chn_sub_expected_outputs_cpp,
default_pb_chn_pub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "mqtt", "pb", "chn"],
},
# -------------------------------------------mqtt_ros2_chn----------------------------------
{
"script_path": [
"./start_examples_plugins_mqtt_plugin_ros2_chn_sub.sh",
"./start_examples_plugins_mqtt_plugin_ros2_chn_pub.sh",
],
"expected_outputs": [
default_ros2_chn_sub_expected_outputs_cpp,
default_ros2_chn_pub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "mqtt", "ros2", "chn"],
},
# --------------------------------------------mqtt_pb_rpc---------------------------------
{
"script_path": [
"./start_examples_plugins_mqtt_plugin_pb_rpc_server.sh",
"./start_examples_plugins_mqtt_plugin_pb_rpc_client.sh",
],
"expected_outputs": [
default_pb_rpc_srv_expected_outputs_cpp,
default_pb_rpc_cli_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "mqtt", "pb", "rpc"],
},
# ---------------------------------------------mqtt_ros2_rpc---------------------------------
{
"script_path": [
"./start_examples_plugins_mqtt_plugin_ros2_rpc_server.sh",
"./start_examples_plugins_mqtt_plugin_ros2_rpc_client.sh",
],
"expected_outputs": [
default_ros2_rpc_srv_expected_outputs_cpp,
default_ros2_rpc_cli_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "mqtt", "ros2", "rpc"],
},
# ------------------------------------------ros2_pb_chn--------------------------------------
{
"script_path": [
"./start_examples_plugins_ros2_plugin_pb_chn_sub.sh",
"./start_examples_plugins_ros2_plugin_pb_chn_pub.sh",
],
"expected_outputs": [
default_pb_chn_sub_expected_outputs_cpp,
default_pb_chn_pub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "ros2_plugin", "pb", "chn"],
},
# -------------------------------------------ros2_ros2_chn------------------------------------
{
"script_path": [
"./start_examples_plugins_ros2_plugin_ros2_chn_sub.sh",
"./start_examples_plugins_ros2_plugin_ros2_chn_pub.sh",
],
"expected_outputs": [
default_ros2_chn_sub_expected_outputs_cpp,
default_ros2_chn_pub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "ros2_plugin", "ros2", "chn"],
},
# --------------------------------------------ros2_pb_rpc-------------------------------------
{
"script_path": [
"./start_examples_plugins_ros2_plugin_pb_rpc_server.sh",
"./start_examples_plugins_ros2_plugin_pb_rpc_client.sh",
],
"expected_outputs": [
default_pb_rpc_srv_expected_outputs_cpp,
default_pb_rpc_cli_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "ros2_plugin", "pb", "rpc"],
},
# ---------------------------------------------ros2_ros2_rpc------------------------------------
{
"script_path": [
"./start_examples_plugins_ros2_plugin_ros2_rpc_server.sh",
"./start_examples_plugins_ros2_plugin_ros2_rpc_client.sh",
],
"expected_outputs": [
default_ros2_rpc_srv_expected_outputs_cpp,
default_ros2_rpc_cli_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "ros2_plugin", "ros2", "rpc"],
},
# --------------------------------------------grpc_pb_rpc----------------------------------------
{
"script_path": [
"./start_examples_plugins_grpc_plugin_pb_rpc_server.sh",
"./start_examples_plugins_grpc_plugin_pb_rpc_client.sh",
],
"expected_outputs": [
default_pb_rpc_srv_expected_outputs_cpp,
default_pb_rpc_cli_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "grpc", "pb", "rpc"],
"limit": "port:50050",
},
# -------------------------------------------parameter_plugin----------------------------------------
{
"script_path": [
"./start_examples_plugins_parameter_plugin.sh",
"./tools/parameter_plugin_get_parameter.sh",
],
"expected_outputs": [
["Set parameter, key: 'key-1', val: 'val-1'"],
['{"code":0,"msg":"","parameter_value":"val-1"}'],
],
"exit_string": [default_exit_string, ""],
"tags": ["all", "cpp", "plugins", "parameter", "curl"],
"limit": "port:50080",
},
# ------------------------------native(sub)_ros2(pub)_pb_chn----------------------------------
{
"script_path": [
"./native_ros2_pb_chn_subscriber",
"./start_examples_plugins_ros2_plugin_pb_chn_pub.sh",
],
"expected_outputs": [
["msg: count: 1"],
default_pb_chn_pub_expected_outputs_cpp,
],
"exit_string": [
"",
default_exit_string,
],
"tags": ["all", "cpp", "plugins", "ro2_plugin", "pb", "chn", "native"],
},
# ------------------------------native(pub)_ros2(sub)_ros2_chn----------------------------------
{
"script_path": [
"./start_examples_plugins_ros2_plugin_ros2_chn_sub.sh",
"./native_ros2_chn_publisher",
],
"expected_outputs": [
default_ros2_chn_sub_expected_outputs_cpp,
["Publishing msg:"],
],
"exit_string": [
default_exit_string,
"",
],
"tags": ["all", "cpp", "plugins", "ro2_plugin", "ros2", "chn", "native"],
},
# ------------------------------cpp_ececuotr-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_executor.sh",
],
"expected_outputs": [
["Loop count : 1"],
],
"tags": ["all", "cpp", "executor"],
},
# ------------------------------cpp_ececuotr_co-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_executor_co.sh",
],
"expected_outputs": [
["Loop count : 1"],
],
"tags": ["all", "cpp", "executor"],
},
# ------------------------------cpp_ececuotr_co_loop-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_executor_co_loop.sh",
],
"expected_outputs": [
["Loop count : 1"],
],
"tags": ["all", "cpp", "executor"],
},
# ------------------------------cpp_logger-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_logger.sh",
],
"expected_outputs": [
["s = abc, n = 2"],
],
"forbidden_outputs": [
"",
],
"tags": ["all", "cpp", "logger"],
},
# ------------------------------cpp_logger_specify_executor-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_logger_specify_executor.sh",
],
"expected_outputs": [
["s = abc, n = 2"],
],
"forbidden_outputs": [
"",
],
"tags": ["all", "cpp", "logger"],
},
# ------------------------------cpp_logger_rotate_file-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_logger_rotate_file.sh",
],
"expected_outputs": [
["s = abc, n = 2"],
],
"forbidden_outputs": [
"",
],
"tags": ["all", "cpp", "logger"],
},
# ------------------------------cpp_parameter-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_parameter.sh",
],
"expected_outputs": [
["Set parameter, key: 'key-1', val: 'val-1'"],
],
"tags": ["all", "cpp", "parameter"],
},
# ------------------------------cpp_pb_chn-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_pb_chn.sh",
],
"expected_outputs": [
default_pb_chn_pub_expected_outputs_cpp + default_pb_chn_sub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "pb", "chn"],
},
# ------------------------------cpp_pb_chn_pub_app-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_pb_chn_publisher_app.sh",
],
"expected_outputs": [
default_pb_chn_pub_expected_outputs_cpp + default_pb_chn_sub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "pb", "chn", "app"],
},
# ------------------------------cpp_pb_chn_sub_app-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_pb_chn_subscriber_app.sh",
],
"expected_outputs": [
default_pb_chn_pub_expected_outputs_cpp + default_pb_chn_sub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "pb", "chn", "app"],
},
# ------------------------------cpp_pb_chn_single_pkg-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_pb_chn_single_pkg.sh",
],
"expected_outputs": [
default_pb_chn_pub_expected_outputs_cpp + default_pb_chn_sub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "pb", "chn"],
},
# ------------------------------cpp_pb_rpc_async-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_pb_rpc_async.sh",
],
"expected_outputs": [
[
'Client start new rpc call. req: {"msg":"hello world foo, count 1"}',
'return rsp: {"code":"0","msg":"echo hello world foo, count 1"}',
'Client get rpc ret, status: suc, code 0, msg: OK, rsp: {"code":"0","msg":"echo hello world foo, count 1"}',
],
],
"tags": ["all", "cpp", "pb", "rpc"],
},
# ------------------------------cpp_pb_rpc_co-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_pb_rpc_co.sh",
],
"expected_outputs": [
[
"Client start new rpc call.",
'return rsp: {"code":"0","msg":"echo hello world foo, count 1"}',
"Client get rpc ret, status: suc, code 0, msg: OK, rsp:",
],
],
"tags": ["all", "cpp", "pb", "rpc"],
},
# ------------------------------cpp_pb_rpc_future-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_pb_rpc_future.sh",
],
"expected_outputs": [
[
'Client start new rpc call. req: {"msg":"hello world foo, count 1"}',
'return rsp: {"code":"0","msg":"echo hello world foo, count 1"}',
'Client get rpc ret, status: suc, code 0, msg: OK, rsp: {"code":"0","msg":"echo hello world foo, count 1"}',
],
],
"tags": ["all", "cpp", "pb", "rpc"],
},
# ------------------------------cpp_pb_rpc_sync-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_pb_rpc_sync.sh",
],
"expected_outputs": [
[
"Client start new rpc call.",
'return rsp: {"code":"0","msg":"echo hello world foo, count 1"}',
'Client get rpc ret, status: suc, code 0, msg: OK, rsp: {"code":"0","msg":"echo hello world foo, count 1"}',
],
],
"tags": ["all", "cpp", "pb", "rpc"],
},
# ------------------------------cpp_helloworld-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_helloworld.sh",
],
"expected_outputs": [
"will waiting for shutdown",
],
"tags": ["all", "cpp", "helloworld"],
},
# ------------------------------cpp_helloworld_app-----------------------------------------------
{
"script_path": [
"./start_examples_cpp_helloworld_app_mode.sh",
],
"expected_outputs": [
"Start succeeded",
],
"tags": ["all", "cpp", "helloworld", "app"],
},
# ------------------------------cpp_helloworld_app_registration------------------------------------
{
"script_path": [
"./start_examples_cpp_helloworld_app_registration_mode.sh",
],
"expected_outputs": [
"will waiting for shutdown",
],
"tags": ["all", "cpp", "helloworld", "app"],
},
# ------------------------------cpp_record_plugin_imd------------------------------------
{
"script_path": [
"./start_examples_plugins_record_playback_plugin_record_imd.sh",
],
"expected_outputs": [
default_pb_chn_pub_expected_outputs_cpp + default_pb_chn_sub_expected_outputs_cpp,
],
"tags": ["all", "cpp", "plugins", "record_playback"],
},
# ////////////////////////////////////////////////////////////////////////////////////////////////////
# ////////////////////////////////////PYTHON EXAMPLES/////////////////////////////////////////////////
# ///////////////////////////////////////////////////////////////////////////////////////////////////
# ---------------------------------python_helloworld--------------------------------------------
{
"script_path": [
"./start_examples_py_helloworld_app_mode.sh",
],
"expected_outputs": [
["{'key1': 'val1', 'key2': 'val2'}", "Loop count: 1", "Loop count: 2"],
],
"tags": ["all", "python", "helloworld", "app"],
"cwd": py_cwd + "/helloworld",
},
# ---------------------------------python_helloworld_registration--------------------------------
{
"script_path": [
"./start_examples_py_helloworld_registration_mode.sh",
],
"expected_outputs": [
["{'key1': 'val1', 'key2': 'val2'}"] + ["run test task"],
],
"tags": ["all", "python", "helloworld", "app"],
"cwd": py_cwd + "/helloworld",
},
# ---------------------------------python_http_pb_rpc----------------------------------------
{
"script_path": [
"./start_examples_py_pb_rpc_http_server.sh",
"./start_examples_py_pb_rpc_http_client.sh",
],
"expected_outputs": [
default_pb_rpc_srv_expected_outputs_python,
default_pb_rpc_cli_expected_outputs_python,
],
"tags": ["all", "python", "http", "pb", "rpc"],
"cwd": py_cwd + "/pb_rpc",
"limit": "port:50080",
},
# ---------------------------------python_grpc_pb_rpc----------------------------------------
{
"script_path": [
"./start_examples_py_pb_rpc_grpc_server.sh",
"./start_examples_py_pb_rpc_grpc_client.sh",
],
"expected_outputs": [
default_pb_rpc_srv_expected_outputs_python,
default_pb_rpc_cli_expected_outputs_python,
],
"tags": ["all", "python", "grpc", "pb", "rpc"],
"cwd": py_cwd + "/pb_rpc",
"limit": "port:50050",
},
# ---------------------------------python_ros_plugin_pb_rpc----------------------------------------
{
"script_path": [
"./start_examples_py_pb_rpc_ros2_server.sh",
"./start_examples_py_pb_rpc_ros2_client.sh",
],
"expected_outputs": [
default_pb_rpc_srv_expected_outputs_python,
default_pb_rpc_cli_expected_outputs_python,
],
"tags": ["all", "python", "ros_pluginc", "pb", "rpc"],
"cwd": py_cwd + "/pb_rpc",
},
# ---------------------------------python_http_pb_chn----------------------------------------
{
"script_path": [
"./start_examples_py_pb_chn_http_sub.sh",
"./start_examples_py_pb_chn_http_pub.sh",
],
"expected_outputs": [
default_pb_chn_sub_expected_outputs_python,
default_pb_chn_pub_expected_outputs_python,
],
"tags": ["all", "python", "http", "pb", "chn"],
"cwd": py_cwd + "/pb_chn",
"limit": "port:50080",
},
# ---------------------------------python_ros2_plugin_pb_chn----------------------------------------
{
"script_path": [
"./start_examples_py_pb_chn_ros2_sub.sh",
"./start_examples_py_pb_chn_ros2_pub.sh",
],
"expected_outputs": [
default_pb_chn_sub_expected_outputs_python,
default_pb_chn_pub_expected_outputs_python,
],
"tags": ["all", "python", "ros2_plugin", "pb", "chn"],
"cwd": py_cwd + "/pb_chn",
},
# ---------------------------------python_parameter----------------------------------------
{
"script_path": [
"./start_examples_py_parameter.sh",
],
"expected_outputs": [
[
"Start SetParameterLoop.",
"SetParameterLoop count: 1 -------------------------",
"Set parameter, key: 'key-1', val: 'val-1'",
"Start GetParameterLoop.",
"GetParameterLoop count: 1 -------------------------",
"Get parameter, key: 'key-1', val: 'val-1'",
],
],
"tags": ["all", "python", "parameter"],
"cwd": py_cwd + "/parameter",
},
]

345
src/examples/utils/run_all_example.py Normal file
View File

@ -0,0 +1,345 @@
# Copyright (c) 2023, AgiBot Inc.
# All rights reserved.
import argparse
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
from example_items import *
from multiprocessing import Process, Queue
import signal
import subprocess
import tempfile
import threading
import time
from typing import Dict, List, Tuple
import sys
class ExampleRunner:
def __init__(self):
# initial operations
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"\n{YELLOW}{BOLD}Running all examples on {now}...{RESET}\n")
# initial member variables
self.args = self.parse_args() # parse command line arguments
self.test_start_time = time.time()
self.max_threads = self.args.parallel_num # maximum number of threads to use
self.item_results = {} # store test results for each item
self.lock_dict = {} # store lock for some limisted items
if self.args.save is not None:
self.check_and_create_directory(self.args.save) # todo ...
subprocess.run(
["pip3", "install", "./aimrt_py_pkg/dist/aimrt_py-0.9.0-cp310-cp310-linux_x86_64.whl"],
cwd=defualt_build_path,
)
subprocess.run(
["bash", os.path.join("build_examples_py_pb_rpc.sh")],
cwd=os.path.join(py_cwd, "pb_rpc"),
)
subprocess.run(
["bash", os.path.join("build_examples_py_pb_chn.sh")],
cwd=os.path.join(py_cwd, "pb_chn"),
)
def check_and_create_directory(self, test_log_save_path: str) -> None:
if test_log_save_path and not os.path.exists(test_log_save_path):
os.makedirs(test_log_save_path)
def update_progress(self, progress):
sys.stdout.write("\r" + self.draw_progress_bar(progress))
sys.stdout.flush()
def draw_progress_bar(self, progress: float, total_width: int = 50) -> str:
progress = max(0, min(1, progress))
completed_width = int(total_width * progress)
bar = "█" * completed_width + "-" * (total_width - completed_width)
percent = progress * 100
return f"{BRIGHT_GREEN}{percent:.1f}% [{bar}]{RESET}"
def parse_args(self):
parser = argparse.ArgumentParser(description="Run Python tests")
parser.add_argument("-p", "--print-output", action="store_true", help="Print test output", default=False)
parser.add_argument("-t", "--test", nargs="+", type=str, help="Test name", default=["all"])
parser.add_argument("-i", "--ignore", nargs="+", type=str, help="Ignore test name", default=None)
parser.add_argument("-s", "--save", nargs="?", default=None, const=default_save_path, help="Save test log.")
parser.add_argument("-n", "--parallel_num", type=int, help="Number of parallel tests", default=10)
args = parser.parse_args()
return args
def generate_test_report(self, test_results: Dict[str, TestResult]) -> str:
total_tests = len(test_results)
successful_tests = sum(1 for result in test_results.values() if result == TestResult.SUCCESS)
not_found_tests = sum(1 for result in test_results.values() if result == TestResult.EXPECTED_OUTPUT_NOT_FOUND)
forbidden_tests = sum(1 for result in test_results.values() if result == TestResult.FORBIDDEN_OUTPUT_FOUND)
exit_failed_tests = sum(1 for result in test_results.values() if result == TestResult.EXIT_STRING_NOT_FOUND)
not_run_tests = sum(1 for result in test_results.values() if result is None)
width = 65
report = f"""
{CYAN}{BOLD}
_____ _ ____ _
|_ _|__ ___| |_ | _ \\ ___ _ __ ___ _ __| |_ _
| |/ _ \\/ __| __| | |_) / _ \\ '_ \\ / _ \\| '__| __(_)
| | __/\\__ \\ |_ | _ < __/ |_) | (_) | | | |_ _
|_|\\___||___/\\__| |_| \\_\\___| .__/ \\___/|_| \\__(_)
|_|
{RESET}
{YELLOW}{BOLD}► Overall Result:{RESET}
{WHITE}{'Total tests:':┈<{width}}{CYAN}{total_tests}
{GREEN}{'Successful tests:':┈<{width-4}}{successful_tests}
{RED}{'Failed tests:':┈<{width-4}}{not_found_tests + exit_failed_tests + forbidden_tests}
{YELLOW}{'• Expected Output Not Found:':┈<{width-12}}{not_found_tests}
{MAGENTA}{'• Forbidden Output Found:':┈<{width-12}}{forbidden_tests}
{RED}{'• Exit String Not Found:':┈<{width-12}}{exit_failed_tests}
{BLUE}{'Not run tests:':┈<{width-4}}{not_run_tests}
{YELLOW}{BOLD}► Detailed Results:{RESET}
"""
for test_name, result in test_results.items():
if result == TestResult.SUCCESS:
status = f"{GREEN}✔ Success{RESET}"
elif result == TestResult.EXPECTED_OUTPUT_NOT_FOUND:
status = f"{YELLOW}⚠ Expected Output Not Found{RESET}"
elif result == TestResult.FORBIDDEN_OUTPUT_FOUND:
status = f"{MAGENTA}✘ Forbidden Output Found{RESET}"
elif result == TestResult.EXIT_STRING_NOT_FOUND:
status = f"{RED}☹ Exit String Not Found{RESET}"
else: # result is None
status = f"{BLUE}- Not Run{RESET}"
report += f" {CYAN}•{RESET} {test_name:<65} {status}\n"
success_rate = (
(successful_tests / (total_tests - not_run_tests)) * 100 if (total_tests - not_run_tests) > 0 else 0
)
overall_success_rate = (successful_tests / total_tests) * 100 if total_tests > 0 else 0
report += f"\n{YELLOW} Success Rate (excluding not run): {WHITE}{success_rate:.2f}%{RESET}"
report += f"\n{YELLOW} Overall Success Rate: {WHITE}{overall_success_rate:.2f}%{RESET}"
return report
def check_item_format(self, item: dict) -> None:
task_num_for_current_item = 0
# check and some default parameters to be added
if "script_path" not in item:
raise ValueError("script_path is required in item")
else:
task_num_for_current_item = len(item["script_path"])
if "expected_outputs" not in item:
raise ValueError("expected_outputs is required in item")
else:
if len(item["expected_outputs"]) != task_num_for_current_item:
raise ValueError("expected_outputs should have the same length as script_path")
if "forbidden_outputs" not in item:
item["forbidden_outputs"] = [default_forbidden_outputs] * task_num_for_current_item
else:
if len(item["forbidden_outputs"]) != task_num_for_current_item:
raise ValueError("forbidden_outputs should have the same length as script_path")
if "exit_string" not in item:
item["exit_string"] = [default_exit_string] * task_num_for_current_item
else:
if len(item["exit_string"]) != task_num_for_current_item:
raise ValueError("exit_string should have the same length as script_path")
if "timeout" not in item:
item["timeout"] = default_timeout
# add default parameters:build directory path, if not exist
if "cwd" not in item:
item["cwd"] = defualt_build_path
if "limit" in item and item["limit"] not in self.lock_dict:
self.lock_dict[item["limit"]] = threading.Lock()
def run_task_with_timeout(self, script_path: str, cwd: str, running_sec: int, wait_sec: int = 5) -> Tuple[str, str]:
with tempfile.NamedTemporaryFile(mode="w+", encoding="latin-1", suffix=".log", delete=False) as temp_file:
process = subprocess.Popen(
script_path,
stdout=temp_file,
stderr=temp_file,
text=True,
shell=True,
cwd=cwd,
bufsize=1,
universal_newlines=True,
preexec_fn=os.setsid,
)
try:
process.wait(timeout=running_sec)
except subprocess.TimeoutExpired:
# Send SIGTERM signal to the process group
os.killpg(os.getpgid(process.pid), signal.SIGINT)
# Wait for a short time to allow the process to terminate
time.sleep(wait_sec)
if process.poll() is None:
# If the process still hasn't terminated, forcefully terminate it
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
# Ensure the process has ended
process.wait()
# Flush the file buffer
temp_file.flush()
os.fsync(temp_file.fileno())
# Read the log file content
temp_file.seek(0)
log_content = temp_file.read()
os.remove(temp_file.name)
return log_content
def check_result(
self,
log_content: str,
expected_outputs: List[str],
forbidden_outputs: List[str],
exit_str: str,
) -> TestResult:
# prepare expected outputs
expected_outputs_lines = []
for expected_output in expected_outputs:
expected_outputs_lines.extend(expected_output.splitlines())
log_lines = log_content.splitlines()
expected_index = 0
for log_line in log_lines:
if expected_index < len(expected_outputs_lines) and expected_outputs_lines[expected_index] in log_line:
expected_index += 1
# check expected outputs
all_expected_found = expected_index == len(expected_outputs_lines)
if not all_expected_found:
# print(f"{RED}Expected outputs not found: {expected_outputs_lines[expected_index:]}{RESET}")
return TestResult.EXPECTED_OUTPUT_NOT_FOUND
# check forbidden outputs
forbidden_found = any(forbidden_output in log_content for forbidden_output in forbidden_outputs)
if forbidden_found:
# print(f"{RED}Forbidden output found in log content{RESET}")
return TestResult.FORBIDDEN_OUTPUT_FOUND
# check exit string
exit_found = exit_str in log_content
if not exit_found:
# print(f"{RED}Exit string not found: {exit_str}{RESET}")
return TestResult.EXIT_STRING_NOT_FOUND
return TestResult.SUCCESS
def run_and_store_task_result(self, script_path: str, timeout: int, cwd: str, queue: Queue):
log_content = self.run_task_with_timeout(script_path, cwd, timeout)
queue.put((script_path, log_content))
def find_element_index(self, lst, element):
try:
index = lst.index(element)
return index
except ValueError:
return -1
def run_item(self, item):
self.check_item_format(item)
process_list = []
output_queue = Queue()
# if has limits, then acquire lock before running
if "limit" in item and item["limit"] in self.lock_dict:
self.lock_dict[item["limit"]].acquire()
for i in range(len(item["script_path"])):
dt = (
len(item["script_path"]) - 1 - i
) * 2 # this is to ensure sub / srv is close afert pub / cli (todo: better way to do this)
self.item_results[item["script_path"][i]] = None
process = Process(
target=self.run_and_store_task_result,
args=(item["script_path"][i], item["timeout"] + dt, item["cwd"], output_queue),
)
process.start()
process_list.append(process)
time.sleep(0.1)
# wait for all processes to complete
for process in process_list:
process.join()
# when all processes are done, release lock if has limits
if "limit" in item and item["limit"] in self.lock_dict:
self.lock_dict[item["limit"]].release()
# from msg queue to get output
result_dict = {}
while not output_queue.empty():
script_path, log_content = output_queue.get()
result_dict[script_path] = log_content
if self.args.print_output:
print(f"\n{CYAN}{BOLD}Output of {script_path}:{RESET}\n{log_content}")
for script_path, log_content in result_dict.items():
idx = self.find_element_index(item["script_path"], script_path)
self.item_results[script_path] = self.check_result(
log_content, item["expected_outputs"][idx], item["forbidden_outputs"][idx], item["exit_string"][idx]
)
# if need to save log when test failed, save it
if self.args.save and self.item_results[script_path] != TestResult.SUCCESS:
with open(f"{self.args.save}/{os.path.basename(script_path)}.log", "w") as f:
f.write(">> **expected outputs:**\n" + "\n".join(item["expected_outputs"][idx]))
f.write("\n\n>> **forbidden_outputs:**\n" + "\n".join(item["forbidden_outputs"][idx]))
f.write("\n\n>> **running log:**\n")
f.write(log_content)
def run(self) -> None:
has_running_num = 0
# run all examples in parallel
with ThreadPoolExecutor(max_workers=self.max_threads) as executor:
futures = {} # store all futures to wait for completion
for idx, test_item in enumerate(test_items):
# filter out examples which are not desired to run
if "tags" in test_item:
if not all(item in test_item["tags"] for item in self.args.test):
continue
if self.args.ignore and any(item in test_item["tags"] for item in self.args.ignore):
continue
futures[
executor.submit(
self.run_item,
item=test_item,
)
] = idx
has_running_num += 1
# wait for all tests to complete
finished_num = 0
self.update_progress(0)
for future in as_completed(futures):
if has_running_num == 0:
break
finished_num += 1
self.update_progress(finished_num / has_running_num)
time.sleep(0.1)
future.result()
def __del__(self):
# print test report
report = self.generate_test_report(self.item_results)
print(report)
# calculate total running time
print(f"\n{YELLOW}{BOLD}Test all examples finished, consuming {time.time() - self.test_start_time:.2f}s{RESET}")
if __name__ == "__main__":
ExampleRunner().run()

12
src/examples/utils/run_all_example.sh Executable file
View File

@ -0,0 +1,12 @@
#!/bin/bash
# -n <int> : number of iterations (default: 10)
# -s <string> : fault test log output directory (default: "build/test_log")
# -t <string1> <string2> ... : test tags, with logic is "AND" (default: "all")
# -i <string1> <string2> ... : ignore test tags, with logic is "OR" (default: None)
# -p : print test log to console (default: False), we don't suggest to use this option with n > 1
export PYTHONPATH=$(dirname "$(pwd)"):$PYTHONPATH
python3 ./run_all_example.py -n 20 -s "./test_log" -t "python"