refactor(tools): Restructure bagtrans tool and integrate into aimrt_cli (#68)

* CI: change build worlflow image tag from v20240927 to latest

* refactor(tools): Restructure bagtrans tool and integrate into aimrt_cli

- Integrate bagtrans functionality into aimrt_cli
- Update CMakeLists.txt and documentation to reflect these changes

* choro: format the code

* choro: change the document

* choro: remove the bagtrans options

---------

Co-authored-by: yuguanlin <yuguanlin@agibot.com>

CMakeLists.txt

@@ -14,7 +14,6 @@ option(AIMRT_BUILD_EXAMPLES "AimRT build examples." OFF)
 option(AIMRT_BUILD_DOCUMENT "AimRT build document." OFF)
 option(AIMRT_BUILD_RUNTIME "AimRT build runtime." ON)
 option(AIMRT_BUILD_CLI_TOOLS "AimRT build aimrt command line tools." OFF)
-option(AIMRT_BUILD_BAGTRANS "AimRT build bagtrans tool." OFF)
 
 option(AIMRT_BUILD_PYTHON_RUNTIME "AimRT build python runtime." OFF)
 
@@ -144,21 +143,6 @@ if(AIMRT_BUILD_CLI_TOOLS)
   endif()
 endif()
 
-if(AIMRT_BUILD_BAGTRANS)
-  include(CheckPythonPackage)
-  check_python_package(build BUILD_FOUND)
-  check_python_package(wheel WHEEL_FOUND)
-  check_python_package(setuptools SETUPTOOLS_FOUND)
-
-  if(NOT BUILD_FOUND
-     OR NOT WHEEL_FOUND
-     OR NOT SETUPTOOLS_FOUND)
-    set(AIMRT_BUILD_BAGTRANS OFF)
-    message(WARNING "Can not find build, wheel or setuptools in your python environment, will not compile bagtrans tool!")
-    message(WARNING "Try to install build, wheel and setuptools by `pip3 install build wheel setuptools --upgrade`")
-  endif()
-endif()
-
 if(AIMRT_BUILD_WITH_PROTOBUF)
   include(GetProtobuf)
   include(ProtobufGenCode)
@@ -300,12 +284,6 @@ if(AIMRT_INSTALL
   install(CODE "execute_process(COMMAND \"${CMAKE_COMMAND}\" --build \"${CMAKE_BINARY_DIR}\" --config ${CMAKE_BUILD_TYPE} --target create_python_pkg)")
 endif()
 
-if(AIMRT_INSTALL
-   AND AIMRT_BUILD_WITH_ROS2
-   AND AIMRT_BUILD_BAGTRANS)
-  install(CODE "execute_process(COMMAND \"${CMAKE_COMMAND}\" --build \"${CMAKE_BINARY_DIR}\" --config ${CMAKE_BUILD_TYPE} --target bagtrans)")
-endif()
-
 # Print all aimrt options
 message("\n  AIMRT CMake Options/Info:")
 get_cmake_property(all_variables VARIABLES)

build.bat

@@ -10,7 +10,6 @@ cmake -B build ^
     -DAIMRT_BUILD_DOCUMENT=ON ^
     -DAIMRT_BUILD_RUNTIME=ON ^
     -DAIMRT_BUILD_CLI_TOOLS=ON ^
-    -DAIMRT_BUILD_BAGTRANS=OFF ^
     -DAIMRT_BUILD_PYTHON_RUNTIME=ON ^
     -DAIMRT_USE_FMT_LIB=ON ^
     -DAIMRT_BUILD_WITH_PROTOBUF=ON ^

build.sh

@@ -12,7 +12,6 @@ cmake -B build \
     -DAIMRT_BUILD_DOCUMENT=ON \
     -DAIMRT_BUILD_RUNTIME=ON \
     -DAIMRT_BUILD_CLI_TOOLS=ON \
-    -DAIMRT_BUILD_BAGTRANS=ON \
     -DAIMRT_BUILD_PYTHON_RUNTIME=ON \
     -DAIMRT_USE_FMT_LIB=ON \
     -DAIMRT_BUILD_WITH_PROTOBUF=ON \

document/sphinx-cn/release_notes/v0_9_0.md

@@ -8,7 +8,7 @@
   - 现在可以传入 zenoh 原生配置；
 - mqtt 新增配置项以支持加密传输；
 - 新增了第三方库 asio，runtime::core 不再引用 boost，改为引用独立的 asio 库，以减轻依赖；
-- 新增 bagtrans 命令行工具，用于将 使用 aimrt record_playback 插件录制的 bag 文件转换为 ros2 的 bag 文件；
+- 新增 aimrt_cli trans 命令，用于将 使用 aimrt record_playback 插件录制的 bag 文件转换为 ros2 的 bag 文件；
 - 新增 Echo 插件，用于回显消息；
 
 **次要修改**：

document/sphinx-cn/tutorials/cli_tool/bagtrans_tool.md

@@ -1,39 +1,25 @@
-# bagtrans 工具
+# 转换 AimRT 的 bag 文件为 ROS2 的 bag 文件
 
 ## 简介
 
-**bagtrans** 是一个 AimRT 官方提供的命令行工具，目前可以将 AimRT **recordplayback** 插件记录的 bag 文件转换为 ROS2 的 bag 文件，其中ros2 原生消息会直接转换，pb 消息会转换为 aimrt 提供的 ros2_plugin_proto 中的 RosMsgWrapper 消息类型。
+**aimrt_cli** 工具可以将 AimRT **recordplayback** 插件记录的 bag 文件转换为 ROS2 的 bag 文件，其中ros2 消息会直接转换，pb 消息则不会转换。
 
-## 安装
+基本使用样例如下：
-
+```
-**bagtrans** 工具是一个基于 Python 开发的小工具，并且依赖于 aimrt 提供的 ros2_plugin_protobuf 消息类型，需要按照以下步骤安装：
+aimrt_cli trans -s [your_aimrtbag_source_dir] -o [your_output_dir]
-
-1. 需要您的 python 环境中安装了 build、wheel 和 setuptools 包，如果未安装，可以使用以下命令安装：
-
-```bash
-pip3 install build wheel setuptools --upgrade
 ```
 
-2. 需要编译生成 aimrt 源码库中的 ros2_plugin_proto 消息类型，并且在 build.sh 文件中确保`AIMRT_BUILD_ROS2_PLUGIN` 、 `AIMRT_BUILD_RECORD_PLAYBACK_PLUGIN` 和 `AIMRT_BUILD_BAGTRANS` 选项为`ON`，执行您 aimrt 源码库中的`build.sh`文件进行编译，编译完成后，在  `<path_to_your_aimrt_src_code>/src/tools/bagtrans/build/dist` 文件夹下可找到编译生成的`whl`文件。
+您也可以使用 `aimrt_cli -h/--help`查看支持的命令行选项。
-
-3. 在终端中执行以下命令:
-```
-cd <path_to_your_aimrt_src_code>/src/tools/bagtrans/build/dist
-pip3 install bagtrans-<version>-py3-none-any.whl
-```
-**bagtrans**工具将会自动安装到您的 python 环境中。使用 `pip list | grep bagtrans`可查看是否安装成功。可使用 `pip uninstall bagtrans`进行卸载。
 
 
 ## 使用说明
 
-**bagtrans** 工具的使用方法如下：
+**aimrt_cli** 工具的使用方法如下：
 
-每次使用前，需要使用命令 `source <path_to_your_aimrt_src_code>/src/tools/bagtrans/build/share/ros2_plugin_proto/local_setup.bash` 以修改环境变量。
+`trans` 命令的使用方法如下：
-
-transbag 命令的使用方法如下：
 
 ```bash
-bagtrans trans -h, --help 会显示参数说明:
+aimrt_cli trans -h, --help 会显示参数说明:
 
 options:
   -h, --help            show this help message and exit
@@ -44,5 +30,3 @@ options:
 ```
 
 其中 `-s` 参数为必填参数，表示 aimrtbag 的源目录，`-o` 参数为必填参数，表示转换后的bag的输出目录，如果输出目录不存在，则会自动创建；如果输出目录存在，则会覆盖。
-
-

document/sphinx-cn/tutorials/cli_tool/cli_tool.md

@@ -7,6 +7,7 @@
 **aimrt_cli**是一个 AimRT 官方提供的命令行工具，目前支持以下功能：
 
 - [为新项目生成脚手架代码](./gen_prj.md)
+- [转换 AimRT 的 bag 文件为 ROS2 的 bag 文件](./bagtrans_tool.md)
 
 您也可以直接执行`aimrt_cli --help`获取内置帮助说明。更多功能敬请期待。
 

src/protocols/CMakeLists.txt

@@ -34,7 +34,7 @@ if(AIMRT_BUILD_WITH_ROS2)
     add_subdirectory(example_ros2)
   endif()
 
-  if(AIMRT_BUILD_ROS2_PLUGIN OR AIMRT_BUILD_BAGTRANS)
+  if(AIMRT_BUILD_ROS2_PLUGIN)
     add_subdirectory(plugins/ros2_plugin_proto)
   endif()
 endif()

src/tools/CMakeLists.txt

@@ -16,10 +16,6 @@ if(AIMRT_BUILD_CLI_TOOLS)
   add_subdirectory(aimrt_cli)
 endif()
 
-if(AIMRT_BUILD_WITH_ROS2 AND AIMRT_BUILD_BAGTRANS)
-  add_subdirectory(bagtrans)
-endif()
-
 if(AIMRT_BUILD_PYTHON_RUNTIME AND AIMRT_BUILD_PYTHON_PACKAGE)
   add_subdirectory(package_aimrt_py)
 endif()

src/tools/aimrt_cli/aimrt_cli/command/gen_command.py

@@ -3,6 +3,7 @@
 
 from aimrt_cli.command import CommandBase
 from aimrt_cli.generator.project_generator import ProjectGenerator
+from aimrt_cli.trans.rosbag_trans import RosbagTrans
 
 
 class GenCommand(CommandBase):
@@ -23,3 +24,6 @@ class GenCommand(CommandBase):
 
         generator = ProjectGenerator(cfg_path=args.project_cfg, output_dir=args.output_dir)
         generator.generate()
+
+        trans = RosbagTrans(args.src_dir, args.output_dir)
+        trans.trans()

src/tools/aimrt_cli/aimrt_cli/command/trans_command.py

@@ -1,8 +1,8 @@
 # Copyright (c) 2023, AgiBot Inc.
 # All rights reserved.
 
-from bagtrans.command import CommandBase
+from aimrt_cli.command import CommandBase
-from bagtrans.trans.rosbag_trans import RosbagTrans
+from aimrt_cli.trans.rosbag_trans import RosbagTrans
 
 
 class TransCommand(CommandBase):

src/tools/aimrt_cli/aimrt_cli/main.py

@@ -3,8 +3,8 @@
 
 import argparse
 
-
 from aimrt_cli.command.gen_command import GenCommand
+from aimrt_cli.command.trans_command import TransCommand
 
 
 def main(description=None):
@@ -12,18 +12,26 @@ def main(description=None):
         description = 'This is the application generation tool for AimRT.'
 
     parser = argparse.ArgumentParser(description=description)
-    parser.add_argument(
+    subparsers = parser.add_subparsers(dest='command')
-        "generate",
+
-        help="generate the configured project, the configuration yaml files is required.",
+    # gen sub command
-        type=str
+    gen_parser = subparsers.add_parser('gen', help='Generate the configured project')
-    )
+    gen_parser.add_argument("-p", "--project_cfg", help="path of the configuration yaml file")
-    # will support more console parameters later.
+    gen_parser.add_argument("-o", "--output_dir", help="directory you want to output your files")
-    gen_command = GenCommand()
+
-    gen_command.add_arguments(parser, "gen")
+    # bag trans sub command
+    trans_parser = subparsers.add_parser('trans', help='Transform bag files')
+    trans_parser.add_argument("-s", "--src_dir", help="aimrtbag source directory")
+    trans_parser.add_argument("-o", "--output_dir", help="directory you want to output your files")
 
     args = parser.parse_args()
-    if args.generate == "gen":
+
-        gen_command.main(args=args)
+    if args.command == 'gen':
+        generator = GenCommand()
+        generator.main(args=args)
+    elif args.command == 'trans':
+        trans = TransCommand()
+        trans.main(args=args)
     else:
         parser.print_help()
         return 0

src/tools/aimrt_cli/aimrt_cli/trans/rosbag_trans.py

@@ -1,15 +1,13 @@
 # Copyright (c) 2024 The AimRT Authors.
 # AimRT is licensed under Mulan PSL v2.
 
-from bagtrans.trans import TransBase
+from aimrt_cli.trans import TransBase
 import os
 import sqlite3
 import shutil
 from pathlib import Path
 import yaml
 from dataclasses import dataclass
-from ros2_plugin_proto.msg import RosMsgWrapper
-import rclpy.serialization
 
 
 class IndentDumper(yaml.Dumper):
@@ -315,50 +313,6 @@ class RosbagTrans(TransBase):
             print(f"Error create metadata table, error: {e}")
             conn.rollback()
 
-    def update_pb_msg(self, conn, cursor):
-        try:
-            cursor.execute("SELECT id, topic_id, timestamp, data FROM messages")
-            rows = cursor.fetchall()
-
-            update_data = []
-            for row in rows:
-                id, topic_id, timestamp, data = row
-
-                if topic_id not in self.topic_info_dict or not self.topic_info_dict[topic_id].msg_type.startswith("pb"):
-                    continue
-
-                msg = RosMsgWrapper()
-                data = [bytes([d]) for d in data]
-
-                msg.data = data
-                msg.serialization_type = "pb"
-
-                serialized_data = rclpy.serialization.serialize_message(msg)
-
-                update_data.append((sqlite3.Binary(serialized_data), id))
-            cursor.executemany("UPDATE messages SET data = ? WHERE id = ?", update_data)
-            conn.commit()
-
-        except Exception as e:
-            print(f"Error updating msg data from pb to RosMsgWrapper, error: {e}")
-            conn.rollback()
-
-        try:
-            cursor.execute("SELECT id, name, type FROM topics")
-            rows = cursor.fetchall()
-            for row in rows:
-                id = row[0]
-                topic_name = row[1]
-                msg_type = row[2]
-                if msg_type.startswith("pb"):
-                    topic_name = encode_topic_name(topic_name, msg_type)
-                    msg_type = "ros2_plugin_proto/msg/RosMsgWrapper"
-                cursor.execute("UPDATE topics SET name = ? , type = ? WHERE id = ?", (topic_name, msg_type, id))
-            conn.commit()
-        except Exception as e:
-            print(f"Error updating topics table data , error: {e}")
-            conn.rollback()
-
     def trans_single_db(self, db_path: Path):
         single_bag_info = SingleBagProcess(self.topic_info_dict, db_path)
         self.all_duration += single_bag_info.duration_nanoseconds
@@ -376,7 +330,6 @@ class RosbagTrans(TransBase):
             self.insert_schema_version(conn, cursor)
             self.insert_metadata_table(conn, cursor)
             self.insert_topics_table(conn, cursor)
-            self.update_pb_msg(conn, cursor)
             self.update_messages_table(conn, cursor)
         except Exception as e:
             print(f"Error updating messages table: {e}")

src/tools/bagtrans/CMakeLists.txt

@@ -1,34 +0,0 @@
-# Copyright (c) 2023, AgiBot Inc.
-# All rights reserved.
-
-# Get all the files in folder
-file(GLOB_RECURSE BAGTRANS_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/bagtrans/*")
-
-set(BAGTRANS_BUILD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/build)
-
-add_custom_target(
-  bagtrans
-  COMMAND ${Python3_EXECUTABLE} -m build --wheel --no-isolation -v
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-  COMMENT "${PYTHON_EXECUTABLE} -m build --wheel --no-isolation -v"
-  DEPENDS ${BAGTRANS_SRC_FILES})
-
-add_custom_target(
-  copy_ros2_plugin_proto_files_bagtrans
-  COMMAND ${CMAKE_COMMAND} -E make_directory ${BAGTRANS_BUILD_DIR}/lib/
-  COMMAND ${CMAKE_COMMAND} -E make_directory ${BAGTRANS_BUILD_DIR}/local/
-  COMMAND ${CMAKE_COMMAND} -E make_directory ${BAGTRANS_BUILD_DIR}/share/
-  COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_INSTALL_PREFIX}/lib/ ${BAGTRANS_BUILD_DIR}/lib/
-  COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_INSTALL_PREFIX}/local/ ${BAGTRANS_BUILD_DIR}/local/
-  COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_INSTALL_PREFIX}/share/ ${BAGTRANS_BUILD_DIR}/share/
-  COMMENT "Copying ROS2 files"
-  DEPENDS aimrt::plugins::ros2_plugin)
-
-add_dependencies(bagtrans copy_ros2_plugin_proto_files_bagtrans)
-
-if(AIMRT_INSTALL)
-  install(
-    DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    DESTINATION bin
-    USE_SOURCE_PERMISSIONS)
-endif()

src/tools/bagtrans/bagtrans/__init__.py

@@ -1,2 +0,0 @@
-# Copyright (c) 2023, AgiBot Inc.
-# All rights reserved.

src/tools/bagtrans/bagtrans/bagtrans.spec

@@ -1,36 +0,0 @@
-# -*- mode: python ; coding: utf-8 -*-
-
-
-a = Analysis(
-    ['main.py'],
-    pathex=[],
-    binaries=[],
-    hiddenimports=['__future__'],
-    hookspath=[],
-    hooksconfig={},
-    runtime_hooks=[],
-    excludes=[],
-    noarchive=False,
-)
-pyz = PYZ(a.pure)
-
-exe = EXE(
-    pyz,
-    a.scripts,
-    a.binaries,
-    a.datas,
-    [],
-    name='bagtrans',
-    debug=False,
-    bootloader_ignore_signals=False,
-    strip=False,
-    upx=True,
-    upx_exclude=[],
-    runtime_tmpdir=None,
-    console=True,
-    disable_windowed_traceback=False,
-    argv_emulation=False,
-    target_arch=None,
-    codesign_identity=None,
-    entitlements_file=None,
-)

src/tools/bagtrans/bagtrans/command/__init__.py

@@ -1,13 +0,0 @@
-# Copyright (c) 2023, AgiBot Inc.
-# All rights reserved.
-
-
-class CommandBase:
-    def __init__(self):
-        super(CommandBase, self).__init__()
-
-    def add_arguments(self, parser, cmd_name):
-        pass
-
-    def main(self, *, args=None):
-        raise NotImplementedError()

src/tools/bagtrans/bagtrans/main.py

@@ -1,30 +0,0 @@
-# Copyright (c) 2023, AgiBot Inc.
-# All rights reserved.
-
-import argparse
-
-from bagtrans.command.trans_command import TransCommand
-
-
-def main(description=None):
-    if description is None:
-        description = 'This is the bagtrans tool.'
-
-    parser = argparse.ArgumentParser(description=description)
-    subparsers = parser.add_subparsers(dest='command', required=True)
-
-    # Create a subparser for the "trans" command
-    trans_parser = subparsers.add_parser('trans', help='translate the aimrtbag to ros2bag')
-    trans_command = TransCommand()
-    trans_command.add_arguments(trans_parser, "trans")
-
-    args = parser.parse_args()
-    if args.command == "trans":
-        trans_command.main(args=args)
-    else:
-        parser.print_help()
-        return 0
-
-
-if __name__ == '__main__':
-    main()

src/tools/bagtrans/bagtrans/requirements.txt

@@ -1,4 +0,0 @@
-# Copyright (c) 2023, AgiBot Inc.
-# All rights reserved.
-
-setuptools>=59.6.0

src/tools/bagtrans/setup.py

@@ -1,23 +0,0 @@
-# Copyright (c) 2023, AgiBot Inc.
-# All rights reserved.
-
-from setuptools import find_packages, setup
-
-package_name = 'bagtrans'
-
-setup(
-    name=package_name,
-    app='bagtrans',
-    version='0.0.1',
-    packages=find_packages(exclude=['test']),
-    author='Yu Guanlin',
-    author_email='yuguanlin@agibot.com',
-    description='transfer aimrt bag file to ros2 bag file',
-    license='',
-    entry_points={
-        'console_scripts': [
-            'bagtrans = bagtrans.main:main',
-        ],
-    },
-    include_package_data=True,
-)

test.bat

@@ -10,7 +10,6 @@ cmake -B build ^
     -DAIMRT_BUILD_DOCUMENT=ON ^
     -DAIMRT_BUILD_RUNTIME=ON ^
     -DAIMRT_BUILD_CLI_TOOLS=ON ^
-    -DAIMRT_BUILD_BAGTRANS=OFF ^
     -DAIMRT_BUILD_PYTHON_RUNTIME=ON ^
     -DAIMRT_USE_FMT_LIB=ON ^
     -DAIMRT_BUILD_WITH_PROTOBUF=ON ^

test.sh

@@ -12,7 +12,6 @@ cmake -B build \
     -DAIMRT_BUILD_DOCUMENT=ON \
     -DAIMRT_BUILD_RUNTIME=ON \
     -DAIMRT_BUILD_CLI_TOOLS=ON \
-    -DAIMRT_BUILD_BAGTRANS=ON \
     -DAIMRT_BUILD_PYTHON_RUNTIME=ON \
     -DAIMRT_USE_FMT_LIB=ON \
     -DAIMRT_BUILD_WITH_PROTOBUF=ON \