From 184a8bacc97d9ca0f1054b723f8a41a8989d0edc Mon Sep 17 00:00:00 2001 From: manx52 Date: Tue, 24 Sep 2024 18:33:58 -0400 Subject: [PATCH 1/4] fixed docker bug minimized the size for docker by removing cache and optimizing which torch is installed --- .dockerignore | 11 +++++----- .idea/misc.xml | 2 +- .idea/soccerbot.iml | 2 +- .idea/vcs.xml | 1 - tools/docker/Dockerfile | 7 ++++--- tools/setup/requirements-gpu.txt | 2 +- tools/setup/requirements.txt | 35 ++------------------------------ 7 files changed, 14 insertions(+), 46 deletions(-) diff --git a/.dockerignore b/.dockerignore index 87faeb72a..83c2ee74b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,15 +10,14 @@ external !external/GameController/protocols/python/gamestate.py !external/CATKIN_IGNORE LICENSE -Dockerfile docker-compose.* +tools/docker/* +compose.* bez1_description/meshes/*.stl bez2_description/meshes/*.stl -soccer_fieldline_detection/media -#soccerbot/bags/* +soccerbot/bags/* !soccerbot/bags/*.sh LICENSE venv/ -#*.webm -#soccer_perception/data/images/* -#soccer_perception/data/videos/* +soccer_perception/data/* +soccer_perception/soccer_localization/data/* diff --git a/.idea/misc.xml b/.idea/misc.xml index 566caa28a..64c2b57d8 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,5 +3,5 @@ - + \ No newline at end of file diff --git a/.idea/soccerbot.iml b/.idea/soccerbot.iml index c39bdf1c4..00c55d199 100644 --- a/.idea/soccerbot.iml +++ b/.idea/soccerbot.iml @@ -11,7 +11,7 @@ - + diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 9436ac6c3..6fe457631 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -12,6 +12,5 @@ - \ No newline at end of file diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 393bde184..d51892b29 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -1,9 +1,10 @@ ARG BASE_IMAGE=utrarobosoccer/noetic + +FROM $BASE_IMAGE AS dependencies ARG INSTALL_CUDA=true ARG ARCHITECTURE=x86_64 ARG OS=ubuntu2004 -FROM $BASE_IMAGE AS dependencies SHELL ["/bin/bash", "-c"] # Install dependencies @@ -64,8 +65,8 @@ USER robosoccer ENV PATH=/home/robosoccer/.local/bin:$PATH COPY tools/setup/requirements.txt tools/setup/requirements-gpu.txt /tmp/setup/ RUN pip install --upgrade pip -RUN pip install -r /tmp/setup/requirements.txt -RUN if [[ "$INSTALL_CUDA" == "true" ]] ; then pip install -r /tmp/setup/requirements-gpu.txt ; fi +RUN pip install -r /tmp/setup/requirements.txt --no-cache-dir +RUN if [[ "$INSTALL_CUDA" == "true" ]] ; then pip install -r /tmp/setup/requirements-gpu.txt --no-cache-dir ; fi # Predownload neural networks RUN mkdir -p /home/robosoccer/.cache/torch/hub/ && \ diff --git a/tools/setup/requirements-gpu.txt b/tools/setup/requirements-gpu.txt index 14fca847c..8a1fcb3a4 100644 --- a/tools/setup/requirements-gpu.txt +++ b/tools/setup/requirements-gpu.txt @@ -1,4 +1,3 @@ - nvidia-cublas-cu12==12.1.3.1 # via # nvidia-cudnn-cu12 @@ -29,6 +28,7 @@ nvidia-nvjitlink-cu12==12.6.68 # nvidia-cusolver-cu12 # nvidia-cusparse-cu12 nvidia-nvtx-cu12==12.1.105 + # via torch torch @ https://download.pytorch.org/whl/cu121/torch-2.3.1%2Bcu121-cp38-cp38-linux_x86_64.whl ; platform_machine == 'x86_64' torchvision @ https://download.pytorch.org/whl/cu121/torchvision-0.18.1%2Bcu121-cp38-cp38-linux_x86_64.whl ; platform_machine == 'x86_64' diff --git a/tools/setup/requirements.txt b/tools/setup/requirements.txt index 7bbabead8..92ebc0968 100644 --- a/tools/setup/requirements.txt +++ b/tools/setup/requirements.txt @@ -379,37 +379,6 @@ numpy==1.24.4 # transforms3d # ultralytics # ultralytics-thop -nvidia-cublas-cu12==12.1.3.1 - # via - # nvidia-cudnn-cu12 - # nvidia-cusolver-cu12 - # torch -nvidia-cuda-cupti-cu12==12.1.105 - # via torch -nvidia-cuda-nvrtc-cu12==12.1.105 - # via torch -nvidia-cuda-runtime-cu12==12.1.105 - # via torch -nvidia-cudnn-cu12==8.9.2.26 - # via torch -nvidia-cufft-cu12==11.0.2.54 - # via torch -nvidia-curand-cu12==10.3.2.106 - # via torch -nvidia-cusolver-cu12==11.4.5.107 - # via torch -nvidia-cusparse-cu12==12.1.0.106 - # via - # nvidia-cusolver-cu12 - # torch -nvidia-nccl-cu12==2.20.5 - # via torch -nvidia-nvjitlink-cu12==12.6.68 - # via - # nvidia-cusolver-cu12 - # nvidia-cusparse-cu12 -nvidia-nvtx-cu12==12.1.105 - # via torch open3d==0.17.0 ; platform_machine == "x86_64" # via soccerbot (pyproject.toml) opencv-python==4.10.0.84 @@ -648,12 +617,12 @@ tomli==2.0.1 # incremental # pytest # soccerbot (pyproject.toml) -torch==2.3.1; platform_machine == 'x86_64' +torch @ https://download.pytorch.org/whl/cpu/torch-2.3.1%2Bcpu-cp38-cp38-linux_x86_64.whl; platform_machine == 'x86_64' # via # torchvision # ultralytics # ultralytics-thop -torchvision==0.18.1 ; platform_machine == 'x86_64' +torchvision @ https://download.pytorch.org/whl/cpu/torchvision-0.18.1%2Bcpu-cp38-cp38-linux_x86_64.whl ; platform_machine == 'x86_64' # via ultralytics tornado==6.4.1 # via From 2a01261a863e31d839bce37d8c195242f4b3832c Mon Sep 17 00:00:00 2001 From: manx52 Date: Wed, 25 Sep 2024 18:11:11 -0400 Subject: [PATCH 2/4] fixed introspection error now docker should be good to go --- compose.yaml | 2 ++ tools/docker/Dockerfile | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/compose.yaml b/compose.yaml index 7acddde51..15757d1fb 100644 --- a/compose.yaml +++ b/compose.yaml @@ -20,6 +20,8 @@ services: extends: file: tools/docker/compose.autonomy.yaml service: soccerbot_no_cuda + environment: + PYTHONPATH: $$PYTHONPATH:/home/robosoccer/.local/lib/python3.8/site-packages:/opt/ros/noetic/lib/python3/dist-packages:/home/robosoccer/catkin_ws/devel/lib/python3/dist-packages soccerbot: <<: *gpu diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index d51892b29..547384de2 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -88,8 +88,10 @@ WORKDIR /home/robosoccer/catkin_ws # Build Python ROS Packages RUN source /opt/ros/noetic/setup.bash && catkin config --cmake-args -DCMAKE_BUILD_TYPE=Debug RUN source /opt/ros/noetic/setup.bash && catkin build --no-status soccerbot -RUN echo "source /home/robosoccer/catkin_ws/devel/setup.bash" >> ~/.bashrc +RUN echo "source /home/robosoccer/ca/opt/ros/noetic/lib/python3/dist-packagestkin_ws/devel/setup.bash" >> ~/.bashrc + +# TODO need another stage for dev or maybe another docker +RUN sudo cp -a /home/robosoccer/catkin_ws/devel/lib/python3/dist-packages/soccer_msgs /opt/ros/noetic/lib/python3/dist-packages/ +RUN sudo cp -a /opt/ros/noetic/lib/python3/dist-packages/. /usr/local/lib/python3.8/dist-packages/ -RUN sudo ln -s /home/robosoccer/catkin_ws/devel/lib/python3/dist-packages/soccer_msgs /opt/ros/noetic/lib/python3/dist-packages/ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/aarch64-linux-gnu/tegra:/usr/local/cuda/targets/aarch64-linux/lib/ -ENV PYTHONPATH=$PYTHONPATH:/home/robosoccer/.local/lib/python3.8/site-packages:/opt/ros/noetic/lib/python3/dist-packages:/home/robosoccer/catkin_ws/devel/lib/python3/dist-packages From efff91ea91c96437512f661d2138445917125e34 Mon Sep 17 00:00:00 2001 From: manx52 Date: Fri, 27 Sep 2024 18:45:38 -0400 Subject: [PATCH 3/4] removed webots as an external module and replaced with a script from bit bots to install. this will make the code a lot more modular and decrease size. Made scripts to install and get the env uptodate so that new members dont have to struggle on installitaion --- .gitmodules | 4 - .idea/misc.xml | 2 +- .idea/soccerbot.iml | 2 +- external/webots | 1 - soccerbot/launch/modules/webots.launch | 2 +- soccerbot/scripts/webots.sh | 7 +- tools/setup/setup.sh | 137 +++++++++++++++++++++++++ tools/setup/setup_webots.sh | 35 +++++++ 8 files changed, 179 insertions(+), 11 deletions(-) delete mode 160000 external/webots create mode 100644 tools/setup/setup.sh create mode 100755 tools/setup/setup_webots.sh diff --git a/.gitmodules b/.gitmodules index 4fafdf74c..d0e4e73cd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule "external/webots"] - path = external/webots - url = git@github.com:utra-robosoccer/webots.git - branch = R2022b [submodule "external/GameController"] path = external/GameController url = git@github.com:RoboCup-Humanoid-TC/GameController.git diff --git a/.idea/misc.xml b/.idea/misc.xml index 64c2b57d8..a9bea494c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,5 +3,5 @@ - + \ No newline at end of file diff --git a/.idea/soccerbot.iml b/.idea/soccerbot.iml index 00c55d199..2237439db 100644 --- a/.idea/soccerbot.iml +++ b/.idea/soccerbot.iml @@ -11,7 +11,7 @@ - + diff --git a/external/webots b/external/webots deleted file mode 160000 index a5ad48b34..000000000 --- a/external/webots +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a5ad48b34b281bc26042c843229514a8fd37b076 diff --git a/soccerbot/launch/modules/webots.launch b/soccerbot/launch/modules/webots.launch index b9d6fc77e..223f1e4e8 100644 --- a/soccerbot/launch/modules/webots.launch +++ b/soccerbot/launch/modules/webots.launch @@ -1,7 +1,7 @@ - + diff --git a/soccerbot/scripts/webots.sh b/soccerbot/scripts/webots.sh index 5e1ce5db5..f175f1124 100755 --- a/soccerbot/scripts/webots.sh +++ b/soccerbot/scripts/webots.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash export GAME_CONTROLLER_HOME=~/catkin_ws/src/soccerbot/external/GameController/ export JAVA_HOME=/usr -cd ~/catkin_ws/src/soccerbot/external/webots || exit 1 -#./webots ./../hlvs_webots/worlds/robocup.wbt -./webots ./../hlvs_webots/worlds/robocup.wbt +export WEBOTS_HOME=/usr/local/webots + +cd ~/catkin_ws/src/soccerbot/external || exit 1 +webots ./hlvs_webots/worlds/robocup.wbt diff --git a/tools/setup/setup.sh b/tools/setup/setup.sh new file mode 100644 index 000000000..2c774afa2 --- /dev/null +++ b/tools/setup/setup.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash + +check_internet_connection () { + if ! ping -q -c 1 -W 1 google.com > /dev/null; then + echo "No internet connection. Please check your internet connection to install the webots simulator." + exit 1 + fi +} + +ask_question() { + while true; do + read -p "$1 [Y/n]: " -n 1 -r response + echo "" + case $response in + [Yy] | "") return 0;; + [Nn]) return 1;; + * ) echo "Please answer yes or no.";; + esac + done +} + +setup_cuda(){ + if (( has_sudo )); then + export OS=ubuntu2004 + export ARCHITECTURE=x86_64 # sbsa for ARM + wget https://developer.download.nvidia.com/compute/cuda/repos/$OS/$ARCHITECTURE/cuda-$OS.pin + sudo mv cuda-$OS.pin /etc/apt/preferences.d/cuda-repository-pin-600 + sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/${OS}/$ARCHITECTURE/3bf863cc.pub + sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/${OS}/$ARCHITECTURE/ /" + sudo apt-get update && sudo apt-get -y install cuda libcudnn9-cuda-12 libcudnn9-dev-cuda-12 libnccl2 libnccl-dev + else + echo "Please install CUDA manually!" + fi + +} + + +setup_ros(){ + if (( has_sudo )); then + echo "Setting up ROS ..." + sudo apt-get install -y python3-pip vim git git-lfs python-is-python3 + pip3 install pre-commit + echo "export PATH=/home/$USER/.local/bin:$PATH" >> ~/.bashrc && source ~/.bashrc + + sudo sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list' + sudo apt install -y curl # if you haven't already installed curl + curl -s https://raw.githubusercontent.com/ros/rosdistro/master/ros.asc | sudo apt-key add - + sudo apt update && sudo apt install -y ros-noetic-desktop-full + echo "source /opt/ros/noetic/setup.bash" >> ~/.bashrc && source ~/.bashrc + sudo apt install -y python3-rosdep python3-rosinstall python3-rosinstall-generator python3-wstool build-essential + sudo rosdep init + rosdep update + + sudo sh \ + -c 'echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -sc` main" \ + > /etc/apt/sources.list.d/ros-latest.list' + wget http://packages.ros.org/ros.key -O - | sudo apt-key add - + sudo apt-get update && sudo apt-get install -y python3-catkin-tools + + + else + echo "Please install ROS manually!" + fi + +} + +setup_repo(){ + if (( has_sudo )); then + echo "Setting up Repo ..." + mkdir -p ~/catkin_ws/src && cd ~/catkin_ws + catkin init + catkin config --cmake-args -DCMAKE_BUILD_TYPE=Debug # For Debug builds + cd src + git clone --recurse-submodules git@github.com:utra-robosoccer/soccerbot.git + git pull + git submodule sync + git submodule update --init --recursive + git-lfs pull + git submodule foreach git-lfs pull + + cd ~/catkin_ws/src/soccerbot/ + sudo apt-get install -y $(cat tools/setup/rosdep.txt) + pip install --upgrade pip + pip install -U setuptools[core] + pip install -r /tools/setup/requirements.txt + if (( wants_cuda )) ; then + pip install -r /tools/setup/requirements-gpu.txt + fi + pre-commit install + rosdep update && rosdep install --from-paths . --ignore-src -r -y + + # Building the webots controllers + cd ~/catkin_ws/src/soccerbot/external/hlvs_webots + pip install -r controllers/referee/requirements.txt + sudo apt-get install protobuf-compiler libprotobuf-dev ant + make + + # Building GameController + cd ~/catkin_ws/src/soccerbot/external/GameController + ant + + mkdir -p /home/$USER/.ros/config && cd /home/$USER/.ros/config + ln -s /opt/ros/noetic/etc/ros/python_logging.conf + + catkin build soccerbot + echo "source /home/$USER/catkin_ws/devel/setup.bash" >> ~/.bashrc && source ~/.bashrc + + else + echo "Please install Repo manually!" + fi + +} + + + + +has_sudo=0 +wants_cuda=0 +if ask_question "Do you have sudo rights?"; then + has_sudo=1 +fi +if ask_question "Do you have a nvidia graphics card and want to install cuda"; then + wants_cuda=1 +fi + +if (( ! has_sudo )); then + echo "Because, you don't have sudo rights, ensure all necessary ROS packages are installed." +fi + +check_internet_connection + +if (( wants_cuda )) ; then + setup_cuda +fi + +setup_ros +setup_repo diff --git a/tools/setup/setup_webots.sh b/tools/setup/setup_webots.sh new file mode 100755 index 000000000..d253b11b4 --- /dev/null +++ b/tools/setup/setup_webots.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -eEo pipefail + +WEBOTS_VERSION="2022b" +WEBOTS_DOWNLOAD_URL="https://github.com/cyberbotics/webots/releases/download/R${WEBOTS_VERSION}/webots_${WEBOTS_VERSION}_amd64.deb" + +check_internet_connection () { + if ! ping -q -c 1 -W 1 google.com > /dev/null; then + echo "No internet connection. Please check your internet connection to install the webots simulator." + exit 1 + fi +} + +# Check if the correct webots simulator WEBOTS_VERSION is installed (apt) +if apt list webots --installed | grep -q "$WEBOTS_VERSION"; then + echo "Webots simulator release $WEBOTS_VERSION is already installed." +else + echo "Webots simulator release $WEBOTS_VERSION is not installed. Installing..." + # Check if we have an internet connection + check_internet_connection + # Check if the url exist + if ! curl --output /dev/null --silent --head --fail "$WEBOTS_DOWNLOAD_URL"; then + echo "Webots download url does not exist. Please check the url and update the 'WEBOTS_DOWNLOAD_URL' variable in the 'make_webots.sh' script." + exit 1 + fi + # Download the webots simulator dep package to temp folder + wget --no-verbose --show-progress "$WEBOTS_DOWNLOAD_URL" -O "/tmp/webots_${WEBOTS_VERSION}.deb" + # Install the webots simulator + sudo apt-get install "/tmp/webots_${WEBOTS_VERSION}.deb" -y + + echo "export WEBOTS_HOME=/usr/local/webots" >> ~/.bashrc && source ~/.bashrc + echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/ros/noetic/lib/:$WEBOTS_HOME/lib/controller" >> ~/.bashrc + echo "export PYTHONPATH=$PYTHONPATH:$WEBOTS_HOME/lib/controller/python38" >> ~/.bashrc && source ~/.bashrc + +fi From 03ef55babb65725c205f8bbb24a13b08550028fd Mon Sep 17 00:00:00 2001 From: manx52 Date: Fri, 27 Sep 2024 18:47:38 -0400 Subject: [PATCH 4/4] Removed .idea files --- .gitignore | 2 +- .idea/codeStyles/Project.xml | 57 --------------- .idea/codeStyles/codeStyleConfig.xml | 5 -- .idea/inspectionProfiles/Project_Default.xml | 69 ------------------- .../inspectionProfiles/profiles_settings.xml | 6 -- .idea/misc.xml | 7 -- .idea/modules.xml | 8 --- .idea/remote-mappings.xml | 16 ----- .idea/soccerbot.iml | 32 --------- .idea/vcs.xml | 16 ----- 10 files changed, 1 insertion(+), 217 deletions(-) delete mode 100644 .idea/codeStyles/Project.xml delete mode 100644 .idea/codeStyles/codeStyleConfig.xml delete mode 100644 .idea/inspectionProfiles/Project_Default.xml delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/remote-mappings.xml delete mode 100644 .idea/soccerbot.iml delete mode 100644 .idea/vcs.xml diff --git a/.gitignore b/.gitignore index e3305e30f..73fbae297 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ # Python compiled files __pycache__/ *.pyc -.idea/ +.idea # webots cache files soccer_webots/protos/*.cache soccer_webots/protos/Soccerbot_meshes/*.cache diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml deleted file mode 100644 index 3e22f4b71..000000000 --- a/.idea/codeStyles/Project.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml deleted file mode 100644 index 79ee123c2..000000000 --- a/.idea/codeStyles/codeStyleConfig.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index a86eb75c1..000000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,69 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2da2..000000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index a9bea494c..000000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 7a45522d0..000000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/remote-mappings.xml b/.idea/remote-mappings.xml deleted file mode 100644 index f97a43e32..000000000 --- a/.idea/remote-mappings.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/soccerbot.iml b/.idea/soccerbot.iml deleted file mode 100644 index 2237439db..000000000 --- a/.idea/soccerbot.iml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 6fe457631..000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file