Merge pull request #2172 from open-mmlab/dev-1.x

Merge dev-1.x into 1.x
open-mmlab · Jan 10, 2023 · 14118a3 · 14118a3
2 parents 20ac077 + 215378c
commit 14118a3
Show file tree

Hide file tree

Showing 359 changed files with 19,960 additions and 5,822 deletions.
diff --git a/.circleci/test.yml b/.circleci/test.yml
@@ -66,11 +66,11 @@ jobs:
             pip install -U openmim
             mim install 'mmcv >= 2.0.0rc1'
             pip install git+ssh://[email protected]/open-mmlab/[email protected]
-            pip install git+https://github.com/open-mmlab/[email protected]
+            pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
             pip install -r requirements.txt
       - when:
           condition:
-            equal: [ "1.12.1", << parameters.torch >> ]
+            equal: [ "1.13.0", << parameters.torch >> ]
           steps:
             - run: pip install timm
       - when:
@@ -124,8 +124,8 @@ jobs:
             docker exec mmaction pip install git+https://[email protected]/open-mmlab/mmengine.git@main
             docker exec mmaction pip install -U openmim
             docker exec mmaction mim install 'mmcv >= 2.0.0rc1'
-            docker exec mmaction pip install git+https://[email protected]/open-mmlab/[email protected]
-            docker exec mmaction pip install git+https://github.com/open-mmlab/[email protected]
+            docker exec mmaction pip install git+https://[email protected]/open-mmlab/mmdetection.git@dev-3.x
+            docker exec mmaction pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
             docker exec mmaction pip install -r requirements.txt
       - when:
           condition:
@@ -177,13 +177,13 @@ workflows:
           name: minimum_version_cpu
           torch: 1.6.0
           torchvision: 0.7.0
-          python: 3.6.9  # The lowest python 3.6.x version available on CircleCI images
+          python: 3.7.4
           requires:
             - lint
       - build_cpu:
           name: maximum_version_cpu
-          torch: 1.12.1
-          torchvision: 0.13.1
+          torch: 1.13.0
+          torchvision: 0.14.0
           python: 3.9.0
           requires:
             - minimum_version_cpu

diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -17,10 +17,6 @@ jobs:
           python-version: 3.7
       - name: Install pre-commit hook
         run: |
-          # markdownlint requires ruby >= 2.7
-          sudo apt-add-repository ppa:brightbox/ruby-ng -y
-          sudo apt-get update
-          sudo apt-get install -y ruby2.7
           pip install pre-commit
           pre-commit install
       - name: Linting

diff --git a/.github/workflows/merge_stage_test.yml b/.github/workflows/merge_stage_test.yml
@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-18.04
     strategy:
       matrix:
-        python-version: [3.6, 3.8, 3.9]
+        python-version: [3.8, 3.9]
         torch: [1.8.1]
         include:
           - torch: 1.8.1
@@ -58,9 +58,9 @@ jobs:
           pip install -U openmim
           mim install 'mmcv >= 2.0.0rc1'
       - name: Install MMDet
-        run: pip install git+https://github.com/open-mmlab/[email protected]
+        run: pip install git+https://github.com/open-mmlab/mmdetection.git@dev-3.x
       - name: Install MMCls
-        run: pip install git+https://github.com/open-mmlab/[email protected]
+        run: pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
       - name: Install PytorchVideo
         run: pip install pytorchvideo
         if: ${{matrix.torchvision == '0.10.0'}}
@@ -126,9 +126,9 @@ jobs:
           pip install -U openmim
           mim install 'mmcv >= 2.0.0rc1'
       - name: Install MMDet
-        run: pip install git+https://github.com/open-mmlab/[email protected]
+        run: pip install git+https://github.com/open-mmlab/mmdetection.git@dev-3.x
       - name: Install MMCls
-        run: pip install git+https://github.com/open-mmlab/[email protected]
+        run: pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
       - name: Install unittest dependencies
         run: pip install -r requirements.txt
       - name: Install PytorchVideo
@@ -189,8 +189,8 @@ jobs:
           pip install git+https://github.com/open-mmlab/mmengine.git@main
           pip install -U openmim
           mim install 'mmcv >= 2.0.0rc1'
-          pip install git+https://github.com/open-mmlab/[email protected]
-          pip install git+https://github.com/open-mmlab/[email protected]
+          pip install git+https://github.com/open-mmlab/mmdetection.git@dev-3.x
+          pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
           pip install -r requirements.txt
       - name: Install PytorchVideo
         run: python -m pip install pytorchvideo
@@ -226,8 +226,8 @@ jobs:
           pip install git+https://github.com/open-mmlab/mmengine.git@main
           pip install -U openmim
           mim install 'mmcv >= 2.0.0rc1'
-          pip install git+https://github.com/open-mmlab/[email protected]
-          pip install git+https://github.com/open-mmlab/[email protected]
+          pip install git+https://github.com/open-mmlab/mmdetection.git@dev-3.x
+          pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
           pip install -r requirements.txt
       - name: Install PytorchVideo
         run: python -m pip install pytorchvideo

diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml
@@ -48,9 +48,9 @@ jobs:
           pip install -U openmim
           mim install 'mmcv >= 2.0.0rc1'
       - name: Install MMDet
-        run: pip install git+https://github.com/open-mmlab/[email protected]
+        run: pip install git+https://github.com/open-mmlab/mmdetection.git@dev-3.x
       - name: Install MMCls
-        run: pip install git+https://github.com/open-mmlab/[email protected]
+        run: pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
       - name: Install unittest dependencies
         run: pip install -r requirements.txt
       - name: Install PytorchVideo
@@ -113,8 +113,8 @@ jobs:
           pip install git+https://github.com/open-mmlab/mmengine.git@main
           pip install -U openmim
           mim install 'mmcv >= 2.0.0rc1'
-          pip install git+https://github.com/open-mmlab/[email protected]
-          pip install git+https://github.com/open-mmlab/[email protected]
+          pip install git+https://github.com/open-mmlab/mmdetection.git@dev-3.x
+          pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
           pip install -r requirements.txt
       - name: Install PytorchVideo
         run: python -m pip install pytorchvideo
@@ -145,7 +145,7 @@ jobs:
       - name: Upgrade pip
         run: |
           python -V
-          pip install pip --upgrade
+          python -m pip install pip --upgrade
       - name: Install librosa and soundfile
         run: python -m pip install librosa soundfile
       - name: Install lmdb
@@ -159,8 +159,8 @@ jobs:
           pip install git+https://github.com/open-mmlab/mmengine.git@main
           pip install -U openmim
           mim install 'mmcv >= 2.0.0rc1'
-          pip install git+https://github.com/open-mmlab/[email protected]
-          pip install git+https://github.com/open-mmlab/[email protected]
+          pip install git+https://github.com/open-mmlab/mmdetection.git@dev-3.x
+          pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
           pip install -r requirements.txt
       - name: Install PytorchVideo
         run: python -m pip install pytorchvideo

diff --git a/.github/workflows/test_mim.yml b/.github/workflows/test_mim.yml
diff --git a/README.md b/README.md
@@ -64,15 +64,21 @@ The 1.x branch works with **PyTorch 1.6+**.
 
 - **Modular design**: We decompose a video understanding framework into different components. One can easily construct a customized video understanding framework by combining different modules.
 
-- **Support four major video understanding tasks**: MMAction2 implements various algorithms for multiple video understanding tasks, including action recognition, action localization, spatio-temporal action detection, and skeleton-based action detection. We support **27** different algorithms and **20** different datasets for the four major tasks.
+- **Support four major video understanding tasks**: MMAction2 implements various algorithms for multiple video understanding tasks, including action recognition, action localization, spatio-temporal action detection, and skeleton-based action detection.
 
 - **Well tested and documented**: We provide detailed documentation and API reference, as well as unit tests.
 
 ## What's New
 
-- (2022-10-11) We support **Video Swin Transformer** on Kinetics400 and additionally train a Swin-L model on Kinetics700 to extract video features for downstream tasks.
+**Release**: v1.0.0rc2 with the following new features:
 
-**Release**: v1.0.0rc1 was released in 14/10/2022. Please refer to [changelog.md](docs/en/notes/changelog.md) for details and release history.
+- We Support Omni-Sourece training on ImageNet and Kinetics datasets.
+- We support exporting spatial-temporal detection models to ONNX.
+- We support **STGCN++** on NTU-RGB+D.
+- We support **MViT V2** on Kinetics 400 and something-V2.
+- We refine our skeleton-based pipelines and support the joint training of multi-stream skeleton information, including **joint, bone, joint-motion, and bone-motion**.
+- We support **VideoMAE** on Kinetics400.
+- We support **C2D** on Kinetics400, achieve 73.57% Top-1 accuracy (higher than 71.8% in the [paper](https://arxiv.org/abs/1711.07971)).
 
 ## Installation
 
@@ -88,17 +94,18 @@ Please refer to [install.md](https://mmaction2.readthedocs.io/en/1.x/get_started
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/c3d/README.md">C3D</a> (CVPR'2014)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/tsn/README.md">TSN</a> (ECCV'2016)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/i3d/README.md">I3D</a> (CVPR'2017)</td>
+    <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/c2d/README.md">C2D</a> (CVPR'2018)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/i3d/README.md">I3D Non-Local</a> (CVPR'2018)</td>
-    <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/r2plus1d/README.md">R(2+1)D</a> (CVPR'2018)</td>
   </tr>
   <tr>
+    <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/r2plus1d/README.md">R(2+1)D</a> (CVPR'2018)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/trn/README.md">TRN</a> (ECCV'2018)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/tsm/README.md">TSM</a> (ICCV'2019)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/tsm/README.md">TSM Non-Local</a> (ICCV'2019)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/slowonly/README.md">SlowOnly</a> (ICCV'2019)</td>
-    <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/slowfast/README.md">SlowFast</a> (ICCV'2019)</td>
   </tr>
   <tr>
+    <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/slowfast/README.md">SlowFast</a> (ICCV'2019)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/csn/README.md">CSN</a> (ICCV'2019)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/tin/README.md">TIN</a> (AAAI'2020)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/tpn/README.md">TPN</a> (CVPR'2020)</td>
@@ -109,6 +116,14 @@ Please refer to [install.md](https://mmaction2.readthedocs.io/en/1.x/get_started
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/tanet/README.md">TANet</a> (ArXiv'2020)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/timesformer/README.md">TimeSformer</a> (ICML'2021)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/swin/README.md">VideoSwin</a> (CVPR'2022)</td>
+    <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/recognition/videomae/README.md">VideoMAE</a> (NeurIPS'2022)</td>
+  </tr>
+  <tr>
+    <td><a href="https://github.com/open-mmlab/mmaction2/blob/dev-1.x/configs/recognition/mvit/README.md">MViT V2</a> (CVPR'2022)</td>
+    <td></td>
+    <td></td>
+    <td></td>
+    <td></td>
   </tr>
   <tr>
     <td colspan="5" style="font-weight:bold;">Action Localization</td>
@@ -137,7 +152,7 @@ Please refer to [install.md](https://mmaction2.readthedocs.io/en/1.x/get_started
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/skeleton/stgcn/README.md">ST-GCN</a> (AAAI'2018)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/skeleton/2s-agcn/README.md">2s-AGCN</a> (CVPR'2019)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/skeleton/posec3d/README.md">PoseC3D</a> (CVPR'2022)</td>
-    <td></td>
+    <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/configs/skeleton/stgcnpp/README.md">STGCN++</a> (ArXiv'2022)</td>
     <td></td>
   </tr>
 </table>
@@ -161,10 +176,10 @@ If you have any feature requests, please feel free to leave a comment in [Issues
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/tools/data/kinetics/README.md">Kinetics-[400/600/700]</a> (<a href="https://deepmind.com/research/open-source/kinetics/">Homepage</a>) (CVPR'2017)</td>
   </tr>
   <tr>
-    <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/tools/data/sthv1/README.md">SthV1</a> (<a href="https://20bn.com/datasets/something-something/v1/">Homepage</a>) (ICCV'2017)</td>
-    <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/tools/data/sthv2/README.md">SthV2</a> (<a href="https://20bn.com/datasets/something-something/">Homepage</a>) (ICCV'2017)</td>
+    <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/tools/data/sthv1/README.md">SthV1</a>  (ICCV'2017)</td>
+    <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/tools/data/sthv2/README.md">SthV2</a> (<a href="https://developer.qualcomm.com/software/ai-datasets/something-something">Homepage</a>) (ICCV'2017)</td>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/tools/data/diving48/README.md">Diving48</a> (<a href="http://www.svcl.ucsd.edu/projects/resound/dataset.html">Homepage</a>) (ECCV'2018)</td>
-    <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/tools/data/jester/README.md">Jester</a> (<a href="https://20bn.com/datasets/jester/v1">Homepage</a>) (ICCV'2019)</td>
+    <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/tools/data/jester/README.md">Jester</a> (<a href="https://developer.qualcomm.com/software/ai-datasets/jester">Homepage</a>) (ICCV'2019)</td>
   </tr>
   <tr>
     <td><a href="https://github.com/open-mmlab/mmaction2/blob/1.x/tools/data/mit/README.md">Moments in Time</a> (<a href="http://moments.csail.mit.edu/">Homepage</a>) (TPAMI'2019)</td>

diff --git a/configs/_base_/default_runtime.py b/configs/_base_/default_runtime.py
@@ -6,7 +6,8 @@
     logger=dict(type='LoggerHook', interval=20, ignore_last=False),
     param_scheduler=dict(type='ParamSchedulerHook'),
     checkpoint=dict(type='CheckpointHook', interval=1, save_best='auto'),
-    sampler_seed=dict(type='DistSamplerSeedHook'))
+    sampler_seed=dict(type='DistSamplerSeedHook'),
+    sync_buffers=dict(type='SyncBuffersHook'))
 
 env_cfg = dict(
     cudnn_benchmark=False,

diff --git a/configs/_base_/models/agcn.py b/configs/_base_/models/agcn.py
diff --git a/configs/_base_/models/c2d_r50.py b/configs/_base_/models/c2d_r50.py
@@ -0,0 +1,20 @@
+model = dict(
+    type='Recognizer3D',
+    backbone=dict(
+        type='C2D',
+        depth=50,
+        pretrained='https://download.pytorch.org/models/resnet50-11ad3fa6.pth',
+        norm_eval=False),
+    cls_head=dict(
+        type='I3DHead',
+        num_classes=400,
+        in_channels=2048,
+        spatial_type='avg',
+        dropout_ratio=0.5,
+        init_std=0.01,
+        average_clips='prob'),
+    data_preprocessor=dict(
+        type='ActionDataPreprocessor',
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        format_shape='NCTHW'))