Add tests before pushing to Docker Hub (#3924)

.github/workflows/docker.yaml

@@ -8,23 +8,48 @@ on:
     branches: [main]
   workflow_dispatch:
     inputs:
-      image:
+      dockerfile:
         type: choice
-        description: Select Docker Image
+        description: Select Dockerfile
         options:
-          - Arm64
+          - Dockerfile-arm64
-          - Jetson
+          - Dockerfile-jetson
-          - Python
+          - Dockerfile-python
-          - CPU
+          - Dockerfile-cpu
-          - GPU
+          - Dockerfile
+      push:
+        type: boolean
+        description: Push image to Docker Hub
+        default: true
 
 jobs:
   docker:
     if: github.repository == 'ultralytics/ultralytics'
-    name: Push Docker image to Docker Hub
+    name: Push
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 5
+      matrix:
+        include:
+          - dockerfile: "Dockerfile-arm64"
+            tags: "latest-arm64"
+            platforms: "linux/arm64"
+          - dockerfile: "Dockerfile-jetson"
+            tags: "latest-jetson"
+            platforms: "linux/arm64"
+          - dockerfile: "Dockerfile-python"
+            tags: "latest-python"
+            platforms: "linux/amd64"
+          - dockerfile: "Dockerfile-cpu"
+            tags: "latest-cpu"
+            platforms: "linux/amd64"
+          - dockerfile: "Dockerfile"
+            tags: "latest"
+            platforms: "linux/amd64"
     steps:
       - name: Checkout repo
+        if: github.event_name == 'push' || github.event.inputs.dockerfile == matrix.dockerfile
         uses: actions/checkout@v3
 
       - name: Set up QEMU
@@ -39,54 +64,30 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Build and push arm64 image
+      - name: Build Image
-        if: github.event_name == 'push' || github.event.inputs.image == 'Arm64'
+        run: |
-        uses: docker/build-push-action@v4
+          docker build --platform ${{ matrix.platforms }} -f docker/${{ matrix.dockerfile }} -t ultralytics/ultralytics:${{ matrix.tags }} .
-        continue-on-error: true
-        with:
-          context: .
-          platforms: linux/arm64
-          file: docker/Dockerfile-arm64
-          push: true
-          tags: ultralytics/ultralytics:latest-arm64
 
-      - name: Build and push Jetson image
+      - name: Run Tests
-        if: github.event_name == 'push' || github.event.inputs.image == 'Jetson'
+        if: matrix.platforms == 'linux/amd64'  # arm64 images not supported on GitHub CI runners
-        uses: docker/build-push-action@v4
+        run: |
-        continue-on-error: true
+          docker run ultralytics/ultralytics:${{ matrix.tags }} /bin/bash -c "pip install pytest && pytest tests"
-        with:
-          context: .
-          platforms: linux/arm64
-          file: docker/Dockerfile-jetson
-          push: true
-          tags: ultralytics/ultralytics:latest-jetson
 
-      - name: Build and push Python image
+      - name: Run Benchmarks
-        if: github.event_name == 'push' || github.event.inputs.image == 'Python'
+        if: matrix.platforms == 'linux/amd64'  # arm64 images not supported on GitHub CI runners
-        uses: docker/build-push-action@v4
+        run: |
-        continue-on-error: true
+          docker run ultralytics/ultralytics:${{ matrix.tags }} yolo benchmark model=yolov8n.pt imgsz=160
-        with:
-          context: .
-          file: docker/Dockerfile-python
-          push: true
-          tags: ultralytics/ultralytics:latest-python
 
-      - name: Build and push CPU image
+      - name: Push Image
-        if: github.event_name == 'push' || github.event.inputs.image == 'CPU'
+        if: github.event_name == 'push' || github.event.inputs.push == true
-        uses: docker/build-push-action@v4
+        run: |
-        continue-on-error: true
+          docker push ultralytics/ultralytics:${{ matrix.tags }}
-        with:
-          context: .
-          file: docker/Dockerfile-cpu
-          push: true
-          tags: ultralytics/ultralytics:latest-cpu
 
-      - name: Build and push GPU image
+      - name: Notify on failure
-        if: github.event_name == 'push' || github.event.inputs.image == 'GPU'
+        if: github.event_name == 'push' && (cancelled() || failure())
-        uses: docker/build-push-action@v4
+        uses: slackapi/slack-github-action@v1.23.0
-        continue-on-error: true
         with:
-          context: .
+          payload: |
-          file: docker/Dockerfile
+            {"text": "<!channel> GitHub Actions error for ${{ github.workflow }} ❌\n\n\n*Repository:* https://github.com/${{ github.repository }}\n*Action:* https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}\n*Author:* ${{ github.actor }}\n*Event:* ${{ github.event_name }}\n"}
-          push: true
+        env:
-          tags: ultralytics/ultralytics:latest
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_YOLO }}

docker/Dockerfile-python

@@ -28,7 +28,7 @@ ADD https://github.com/ultralytics/assets/releases/download/v0.0.0/yolov8n.pt /u
 
 # Install pip packages
 RUN python3 -m pip install --upgrade pip wheel
-RUN pip install --no-cache -e '.[export]' thop py-cpuinfo --extra-index-url https://download.pytorch.org/whl/cpu
+RUN pip install --no-cache -e '.[export]' thop --extra-index-url https://download.pytorch.org/whl/cpu
 
 # Run exports to AutoInstall packages
 WORKDIR /tmp_exports

requirements.txt

@@ -4,8 +4,8 @@
 # Base ----------------------------------------
 matplotlib>=3.2.2
 opencv-python>=4.6.0
-Pillow>=7.1.2
+pillow>=7.1.2
-PyYAML>=5.3.1
+pyyaml>=5.3.1
 requests>=2.23.0
 scipy>=1.4.1
 torch>=1.7.0
@@ -23,7 +23,7 @@ pandas>=1.1.4
 seaborn>=0.11.0
 
 # Export --------------------------------------
-# coremltools>=6.0  # CoreML export
+# coremltools>=6.0,<=6.2  # CoreML export
 # onnx>=1.12.0  # ONNX export
 # onnxsim>=0.4.1  # ONNX simplifier
 # nvidia-pyindex  # TensorRT export
@@ -36,8 +36,9 @@ seaborn>=0.11.0
 
 # Extras --------------------------------------
 psutil  # system utilization
+py-cpuinfo  # display CPU info
 # thop>=0.1.1  # FLOPs computation
 # ipython  # interactive notebook
-# albumentations>=1.0.3
+# albumentations>=1.0.3  # training augmentations
 # pycocotools>=2.0.6  # COCO mAP
 # roboflow

setup.py

@@ -48,9 +48,11 @@ setup(
             'mkdocs-redirects',  # for 301 redirects
             'mkdocs-ultralytics-plugin>=0.0.21',  # for meta descriptions and images, dates and authors
         ],
-        'export': ['coremltools>=6.0,<=6.2', 'openvino-dev>=2023.0',
+        'export': [
-                   'tensorflowjs'],  # automatically installs tensorflow
+            'coremltools>=6.0,<=6.2',
-    },
+            'openvino-dev>=2023.0',
+            'tensorflowjs',  # automatically installs tensorflow
+        ], },
     classifiers=[
         'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',
@@ -58,7 +60,6 @@ setup(
         'Intended Audience :: Science/Research',
         'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',

ultralytics/engine/exporter.py

@@ -465,7 +465,7 @@ class Exporter:
     @try_export
     def export_coreml(self, prefix=colorstr('CoreML:')):
         """YOLOv8 CoreML export."""
-        check_requirements('coremltools>=6.0')
+        check_requirements('coremltools>=6.0,<=6.2')
         import coremltools as ct  # noqa
 
         LOGGER.info(f'\n{prefix} starting export with coremltools {ct.__version__}...')

ultralytics/utils/benchmarks.py

@@ -122,7 +122,7 @@ def benchmark(model=Path(SETTINGS['weights_dir']) / 'yolov8n.pt',
             y.append([name, '✅', round(file_size(filename), 1), round(metric, 4), round(speed, 2)])
         except Exception as e:
             if hard_fail:
-                assert type(e) is AssertionError, f'Benchmark hard_fail for {name}: {e}'
+                assert type(e) is AssertionError, f'Benchmark failure for {name}: {e}'
             LOGGER.warning(f'ERROR ❌️ Benchmark failure for {name}: {e}')
             y.append([name, emoji, round(file_size(filename), 1), None, None])  # mAP, t_inference
 
@@ -139,7 +139,7 @@ def benchmark(model=Path(SETTINGS['weights_dir']) / 'yolov8n.pt',
     if hard_fail and isinstance(hard_fail, float):
         metrics = df[key].array  # values to compare to floor
         floor = hard_fail  # minimum metric floor to pass, i.e. = 0.29 mAP for YOLOv5n
-        assert all(x > floor for x in metrics if pd.notna(x)), f'HARD FAIL: one or more metric(s) < floor {floor}'
+        assert all(x > floor for x in metrics if pd.notna(x)), f'Benchmark failure: metric(s) < floor {floor}'
 
     return df
 

ultralytics/utils/torch_utils.py

@@ -17,7 +17,7 @@ import torch.nn.functional as F
 import torchvision
 
 from ultralytics.utils import DEFAULT_CFG_DICT, DEFAULT_CFG_KEYS, LOGGER, RANK, __version__
-from ultralytics.utils.checks import check_requirements, check_version
+from ultralytics.utils.checks import check_version
 
 try:
     import thop
@@ -54,8 +54,7 @@ def smart_inference_mode():
 
 def get_cpu_info():
     """Return a string with system CPU information, i.e. 'Apple M2'."""
-    check_requirements('py-cpuinfo')
+    import cpuinfo  # pip install py-cpuinfo
-    import cpuinfo  # noqa
 
     k = 'brand_raw', 'hardware_raw', 'arch_string_raw'  # info keys sorted by preference (not all keys always available)
     info = cpuinfo.get_cpu_info()  # info dict