local

Reviewed By: JuanBesa

Differential Revision: D91210167

fbshipit-source-id: a563232f4bc82f6f3b99e53df1c88cf0f39747bb

examples/sam3_agent.ipynb

@@ -1,242 +1,242 @@
 {
- "cells": [
+  "cells": [
-  {
+    {
-   "cell_type": "code",
+      "cell_type": "code",
-   "execution_count": 1,
+      "execution_count": 1,
-   "metadata": {},
+      "metadata": {},
-   "outputs": [],
+      "outputs": [],
-   "source": [
+      "source": [
-    "# Copyright (c) Meta Platforms, Inc. and affiliates."
+        "# Copyright (c) Meta Platforms, Inc. and affiliates."
-   ]
+      ]
-  },
+    },
-  {
+    {
-   "cell_type": "markdown",
+      "cell_type": "markdown",
-   "metadata": {},
+      "metadata": {},
-   "source": [
+      "source": [
-    "# SAM 3 Agent"
+        "# SAM 3 Agent"
-   ]
+      ]
-  },
+    },
-  {
+    {
-   "cell_type": "markdown",
+      "cell_type": "markdown",
-   "metadata": {},
+      "metadata": {},
-   "source": [
+      "source": [
-    "This notebook shows an example of how an MLLM can use SAM 3 as a tool, i.e., \"SAM 3 Agent\", to segment more complex text queries such as \"the leftmost child wearing blue vest\"."
+        "This notebook shows an example of how an MLLM can use SAM 3 as a tool, i.e., \"SAM 3 Agent\", to segment more complex text queries such as \"the leftmost child wearing blue vest\"."
-   ]
+      ]
-  },
+    },
-  {
+    {
-   "cell_type": "markdown",
+      "cell_type": "markdown",
-   "metadata": {},
+      "metadata": {},
-   "source": [
+      "source": [
-    "## Env Setup"
+        "## Env Setup"
-   ]
+      ]
-  },
+    },
-  {
+    {
-   "cell_type": "markdown",
+      "cell_type": "markdown",
-   "metadata": {},
+      "metadata": {},
-   "source": [
+      "source": [
-    "First install `sam3` in your environment using the [installation instructions](https://github.com/facebookresearch/sam3?tab=readme-ov-file#installation) in the repository."
+        "First install `sam3` in your environment using the [installation instructions](https://github.com/facebookresearch/sam3?tab=readme-ov-file#installation) in the repository."
-   ]
+      ]
-  },
+    },
-  {
+    {
-   "cell_type": "code",
+      "cell_type": "code",
-   "execution_count": null,
+      "execution_count": null,
-   "metadata": {},
+      "metadata": {},
-   "outputs": [],
+      "outputs": [],
-   "source": [
+      "source": [
-    "import torch\n",
+        "import torch\n",
-    "# turn on tfloat32 for Ampere GPUs\n",
+        "# turn on tfloat32 for Ampere GPUs\n",
-    "# https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices\n",
+        "# https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices\n",
-    "torch.backends.cuda.matmul.allow_tf32 = True\n",
+        "torch.backends.cuda.matmul.allow_tf32 = True\n",
-    "torch.backends.cudnn.allow_tf32 = True\n",
+        "torch.backends.cudnn.allow_tf32 = True\n",
-    "\n",
+        "\n",
-    "# use bfloat16 for the entire notebook. If your card doesn't support it, try float16 instead\n",
+        "# use bfloat16 for the entire notebook. If your card doesn't support it, try float16 instead\n",
-    "torch.autocast(\"cuda\", dtype=torch.bfloat16).__enter__()\n",
+        "torch.autocast(\"cuda\", dtype=torch.bfloat16).__enter__()\n",
-    "\n",
+        "\n",
-    "# inference mode for the whole notebook. Disable if you need gradients\n",
+        "# inference mode for the whole notebook. Disable if you need gradients\n",
-    "torch.inference_mode().__enter__()"
+        "torch.inference_mode().__enter__()"
-   ]
+      ]
-  },
+    },
-  {
+    {
-   "cell_type": "code",
+      "cell_type": "code",
-   "execution_count": null,
+      "execution_count": null,
-   "metadata": {},
+      "metadata": {},
-   "outputs": [],
+      "outputs": [],
-   "source": [
+      "source": [
-    "import os\n",
+        "import os\n",
-    "\n",
+        "\n",
-    "SAM3_ROOT = os.path.dirname(os.getcwd())\n",
+        "SAM3_ROOT = os.path.dirname(os.getcwd())\n",
-    "os.chdir(SAM3_ROOT)\n",
+        "os.chdir(SAM3_ROOT)\n",
-    "\n",
+        "\n",
-    "# setup GPU to use -  A single GPU is good with the purpose of this demo\n",
+        "# setup GPU to use -  A single GPU is good with the purpose of this demo\n",
-    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
+        "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
-    "_ = os.system(\"nvidia-smi\")"
+        "_ = os.system(\"nvidia-smi\")"
-   ]
+      ]
-  },
+    },
-  {
+    {
-   "cell_type": "markdown",
+      "cell_type": "markdown",
-   "metadata": {},
+      "metadata": {},
-   "source": [
+      "source": [
-    "## Build SAM3 Model"
+        "## Build SAM3 Model"
-   ]
+      ]
-  },
+    },
-  {
+    {
-   "cell_type": "code",
+      "cell_type": "code",
-   "execution_count": null,
+      "execution_count": null,
-   "metadata": {},
+      "metadata": {},
-   "outputs": [],
+      "outputs": [],
-   "source": [
+      "source": [
-    "import sam3\n",
+        "import sam3\n",
-    "from sam3 import build_sam3_image_model\n",
+        "from sam3 import build_sam3_image_model\n",
-    "from sam3.model.sam3_image_processor import Sam3Processor\n",
+        "from sam3.model.sam3_image_processor import Sam3Processor\n",
-    "\n",
+        "\n",
-    "sam3_root = os.path.join(os.path.dirname(sam3.__file__), \"..\")\n",
+        "sam3_root = os.path.dirname(sam3.__file__)\n",
-    "bpe_path = f\"{sam3_root}/assets/bpe_simple_vocab_16e6.txt.gz\"\n",
+        "bpe_path = f\"{sam3_root}/assets/bpe_simple_vocab_16e6.txt.gz\"\n",
-    "model = build_sam3_image_model(bpe_path=bpe_path)\n",
+        "model = build_sam3_image_model(bpe_path=bpe_path)\n",
-    "processor = Sam3Processor(model, confidence_threshold=0.5)"
+        "processor = Sam3Processor(model, confidence_threshold=0.5)"
-   ]
+      ]
-  },
+    },
-  {
+    {
-   "cell_type": "markdown",
+      "cell_type": "markdown",
-   "metadata": {},
+      "metadata": {},
-   "source": [
+      "source": [
-    "## LLM Setup\n",
+        "## LLM Setup\n",
-    "\n",
+        "\n",
-    "Config which MLLM to use, it can either be a model served by vLLM that you launch from your own machine or a model is served via external API. If you want to using a vLLM model, we also provided insturctions below."
+        "Config which MLLM to use, it can either be a model served by vLLM that you launch from your own machine or a model is served via external API. If you want to using a vLLM model, we also provided insturctions below."
-   ]
+      ]
-  },
+    },
-  {
+    {
-   "cell_type": "code",
+      "cell_type": "code",
-   "execution_count": null,
+      "execution_count": null,
-   "metadata": {},
+      "metadata": {},
-   "outputs": [],
+      "outputs": [],
-   "source": [
+      "source": [
-    "LLM_CONFIGS = {\n",
+        "LLM_CONFIGS = {\n",
-    "    # vLLM-served models\n",
+        "    # vLLM-served models\n",
-    "    \"qwen3_vl_8b_thinking\": {\n",
+        "    \"qwen3_vl_8b_thinking\": {\n",
-    "        \"provider\": \"vllm\",\n",
+        "        \"provider\": \"vllm\",\n",
-    "        \"model\": \"Qwen/Qwen3-VL-8B-Thinking\",\n",
+        "        \"model\": \"Qwen/Qwen3-VL-8B-Thinking\",\n",
-    "    }, \n",
+        "    },\n",
-    "    # models served via external APIs\n",
+        "    # models served via external APIs\n",
-    "    # add your own\n",
+        "    # add your own\n",
-    "}\n",
+        "}\n",
-    "\n",
+        "\n",
-    "model = \"qwen3_vl_8b_thinking\"\n",
+        "model = \"qwen3_vl_8b_thinking\"\n",
-    "LLM_API_KEY = \"DUMMY_API_KEY\"\n",
+        "LLM_API_KEY = \"DUMMY_API_KEY\"\n",
-    "\n",
+        "\n",
-    "llm_config = LLM_CONFIGS[model]\n",
+        "llm_config = LLM_CONFIGS[model]\n",
-    "llm_config[\"api_key\"] = LLM_API_KEY\n",
+        "llm_config[\"api_key\"] = LLM_API_KEY\n",
-    "llm_config[\"name\"] = model\n",
+        "llm_config[\"name\"] = model\n",
-    "\n",
+        "\n",
-    "# setup API endpoint\n",
+        "# setup API endpoint\n",
-    "if llm_config[\"provider\"] == \"vllm\":\n",
+        "if llm_config[\"provider\"] == \"vllm\":\n",
-    "    LLM_SERVER_URL = \"http://0.0.0.0:8001/v1\"  # replace this with your vLLM server address as needed\n",
+        "    LLM_SERVER_URL = \"http://0.0.0.0:8001/v1\"  # replace this with your vLLM server address as needed\n",
-    "else:\n",
+        "else:\n",
-    "    LLM_SERVER_URL = llm_config[\"base_url\"]"
+        "    LLM_SERVER_URL = llm_config[\"base_url\"]"
-   ]
+      ]
-  },
+    },
-  {
+    {
-   "cell_type": "markdown",
+      "cell_type": "markdown",
-   "metadata": {},
+      "metadata": {},
-   "source": [
+      "source": [
-    "### Setup vLLM server \n",
+        "### Setup vLLM server \n",
-    "This step is only required if you are using a model served by vLLM, skip this step if you are calling LLM using an API like Gemini and GPT.\n",
+        "This step is only required if you are using a model served by vLLM, skip this step if you are calling LLM using an API like Gemini and GPT.\n",
-    "\n",
+        "\n",
-    "* Install vLLM (in a separate conda env from SAM 3 to avoid dependency conflicts).\n",
+        "* Install vLLM (in a separate conda env from SAM 3 to avoid dependency conflicts).\n",
-    "  ```bash\n",
+        "  ```bash\n",
-    "    conda create -n vllm python=3.12\n",
+        "    conda create -n vllm python=3.12\n",
-    "    pip install vllm --extra-index-url https://download.pytorch.org/whl/cu128\n",
+        "    pip install vllm --extra-index-url https://download.pytorch.org/whl/cu128\n",
-    "  ```\n",
+        "  ```\n",
-    "* Start vLLM server on the same machine of this notebook\n",
+        "* Start vLLM server on the same machine of this notebook\n",
-    "  ```bash\n",
+        "  ```bash\n",
-    "    # qwen 3 VL 8B thinking\n",
+        "    # qwen 3 VL 8B thinking\n",
-    "    vllm serve Qwen/Qwen3-VL-8B-Thinking --tensor-parallel-size 4 --allowed-local-media-path / --enforce-eager --port 8001\n",
+        "    vllm serve Qwen/Qwen3-VL-8B-Thinking --tensor-parallel-size 4 --allowed-local-media-path / --enforce-eager --port 8001\n",
-    "  ```"
+        "  ```"
-   ]
+      ]
-  },
+    },
-  {
+    {
-   "cell_type": "markdown",
+      "cell_type": "markdown",
-   "metadata": {},
+      "metadata": {},
-   "source": [
+      "source": [
-    "## Run SAM3 Agent Inference"
+        "## Run SAM3 Agent Inference"
-   ]
+      ]
-  },
+    },
-  {
+    {
-   "cell_type": "code",
+      "cell_type": "code",
-   "execution_count": null,
+      "execution_count": null,
-   "metadata": {},
+      "metadata": {},
-   "outputs": [],
+      "outputs": [],
-   "source": [
+      "source": [
-    "from functools import partial\n",
+        "from functools import partial\n",
-    "from IPython.display import display, Image\n",
+        "from IPython.display import display, Image\n",
-    "from sam3.agent.client_llm import send_generate_request as send_generate_request_orig\n",
+        "from sam3.agent.client_llm import send_generate_request as send_generate_request_orig\n",
-    "from sam3.agent.client_sam3 import call_sam_service as call_sam_service_orig\n",
+        "from sam3.agent.client_sam3 import call_sam_service as call_sam_service_orig\n",
-    "from sam3.agent.inference import run_single_image_inference"
+        "from sam3.agent.inference import run_single_image_inference"
-   ]
+      ]
-  },
+    },
-  {
+    {
-   "cell_type": "code",
+      "cell_type": "code",
-   "execution_count": null,
+      "execution_count": null,
-   "metadata": {
+      "metadata": {
-    "output": {
+        "output": {
-     "id": 689664053567678,
+          "id": 689664053567678,
-     "loadingStatus": "loaded"
+          "loadingStatus": "loaded"
+        }
+      },
+      "outputs": [],
+      "source": [
+        "# prepare input args and run single image inference\n",
+        "image = \"assets/images/test_image.jpg\"\n",
+        "prompt = \"the leftmost child wearing blue vest\"\n",
+        "image = os.path.abspath(image)\n",
+        "send_generate_request = partial(send_generate_request_orig, server_url=LLM_SERVER_URL, model=llm_config[\"model\"], api_key=llm_config[\"api_key\"])\n",
+        "call_sam_service = partial(call_sam_service_orig, sam3_processor=processor)\n",
+        "output_image_path = run_single_image_inference(\n",
+        "    image, prompt, llm_config, send_generate_request, call_sam_service,\n",
+        "    debug=True, output_dir=\"agent_output\"\n",
+        ")\n",
+        "\n",
+        "# display output\n",
+        "if output_image_path is not None:\n",
+        "    display(Image(filename=output_image_path))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": []
+    }
+  ],
+  "metadata": {
+    "fileHeader": "",
+    "fileUid": "be59e249-6c09-4634-a9e7-1f06fd233c42",
+    "isAdHoc": false,
+    "kernelspec": {
+      "display_name": "Python 3 (ipykernel)",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.12.11"
     }
-   },
-   "outputs": [],
-   "source": [
-    "# prepare input args and run single image inference\n",
-    "image = \"assets/images/test_image.jpg\"\n",
-    "prompt = \"the leftmost child wearing blue vest\"\n",
-    "image = os.path.abspath(image)\n",
-    "send_generate_request = partial(send_generate_request_orig, server_url=LLM_SERVER_URL, model=llm_config[\"model\"], api_key=llm_config[\"api_key\"])\n",
-    "call_sam_service = partial(call_sam_service_orig, sam3_processor=processor)\n",
-    "output_image_path = run_single_image_inference(\n",
-    "    image, prompt, llm_config, send_generate_request, call_sam_service, \n",
-    "    debug=True, output_dir=\"agent_output\"\n",
-    ")\n",
-    "\n",
-    "# display output\n",
-    "if output_image_path is not None:\n",
-    "    display(Image(filename=output_image_path))"
-   ]
   },
-  {
+  "nbformat": 4,
-   "cell_type": "code",
+  "nbformat_minor": 2
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "fileHeader": "",
-  "fileUid": "be59e249-6c09-4634-a9e7-1f06fd233c42",
-  "isAdHoc": false,
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.12.11"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
 }

pyproject.toml

@@ -26,7 +26,7 @@ classifiers = [
 ]
 dependencies = [
     "timm>=1.0.17",
-    "numpy==1.26",
+    "numpy>=1.26,<2",
     "tqdm",
     "ftfy==6.1.1",
     "regex",
@@ -82,8 +82,12 @@ train = [
 "Homepage" = "https://github.com/facebookresearch/sam3"
 "Bug Tracker" = "https://github.com/facebookresearch/sam3/issues"
 
-[tool.setuptools]
+[tool.setuptools.packages.find]
-packages = ["sam3", "sam3.model"]
+include = ["sam3*"]
+exclude = ["build*", "scripts*", "examples*"]
+
+[tool.setuptools.package-data]
+sam3 = ["assets/*.txt.gz"]
 
 [tool.setuptools.dynamic]
 version = {attr = "sam3.__version__"}

sam3/__init__.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 from .model_builder import build_sam3_image_model
 
 __version__ = "0.1.0"

sam3/agent/__init__.py

@@ -1 +1,3 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+# pyre-unsafe

sam3/agent/agent_core.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import copy
 import json
 import os
@@ -294,9 +296,9 @@ def agent_inference(
             assert LATEST_SAM3_TEXT_PROMPT != ""
 
             # Make sure that the last message is a image
-            assert (
+            assert messages[-1]["content"][1]["type"] == "image", (
-                messages[-1]["content"][1]["type"] == "image"
+                "Second content element should be an image"
-            ), "Second content element should be an image"
+            )
             messages.pop()  # Remove the last user message
             # Add simplified replacement message
             simplified_message = {
@@ -316,7 +318,7 @@ def agent_inference(
 
             # MLLM check the mask one by one
             for i in range(num_masks):
-                print(f"🔍 Checking mask {i+1}/{num_masks}...")
+                print(f"🔍 Checking mask {i + 1}/{num_masks}...")
                 image_w_mask_i, image_w_zoomed_in_mask_i = visualize(current_outputs, i)
 
                 image_w_zoomed_in_mask_i_path = os.path.join(
@@ -361,7 +363,7 @@ def agent_inference(
                     raise ValueError(
                         "Generated text is None, which is unexpected. Please check the Qwen server and the input parameters."
                     )
-                print(f"Generated text for mask {i+1}: {checking_generated_text}")
+                print(f"Generated text for mask {i + 1}: {checking_generated_text}")
                 verdict = (
                     checking_generated_text.split("<verdict>")[-1]
                     .split("</verdict>")[0]
@@ -369,11 +371,11 @@ def agent_inference(
                 )
                 if "Accept" in verdict:
                     assert not "Reject" in verdict
-                    print(f"Mask {i+1} accepted, keeping it in the outputs.")
+                    print(f"Mask {i + 1} accepted, keeping it in the outputs.")
                     masks_to_keep.append(i)
                 elif "Reject" in verdict:
                     assert not "Accept" in verdict
-                    print(f"Mask {i+1} rejected, removing it from the outputs.")
+                    print(f"Mask {i + 1} rejected, removing it from the outputs.")
                 else:
                     raise ValueError(
                         f"Unexpected verdict in generated text: {checking_generated_text}. Expected 'Accept' or 'Reject'."
@@ -395,7 +397,7 @@ def agent_inference(
                 sam_output_dir, rf"{LATEST_SAM3_TEXT_PROMPT}.png"
             ).replace(
                 ".png",
-                f"_selected_masks_{'-'.join(map(str, [i+1 for i in masks_to_keep]))}.png".replace(
+                f"_selected_masks_{'-'.join(map(str, [i + 1 for i in masks_to_keep]))}.png".replace(
                     "/", "_"
                 ),
             )

sam3/agent/client_llm.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import base64
 import os
 from typing import Any, Optional

sam3/agent/client_sam3.py

@@ -1,11 +1,12 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import json
 import os
 
 import torch
 from PIL import Image
-
 from sam3.model.box_ops import box_xyxy_to_xywh
 from sam3.train.masks_ops import rle_encode
 

sam3/agent/helpers/__init__.py

@@ -1 +1,3 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+# pyre-unsafe

sam3/agent/helpers/boxes.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import math
 from enum import IntEnum, unique
 from typing import List, Tuple, Union
@@ -82,9 +84,9 @@ class BoxMode(IntEnum):
         ], "Relative mode not yet supported!"
 
         if from_mode == BoxMode.XYWHA_ABS and to_mode == BoxMode.XYXY_ABS:
-            assert (
+            assert arr.shape[-1] == 5, (
-                arr.shape[-1] == 5
+                "The last dimension of input shape must be 5 for XYWHA format"
-            ), "The last dimension of input shape must be 5 for XYWHA format"
+            )
             original_dtype = arr.dtype
             arr = arr.double()
 
@@ -242,9 +244,9 @@ class Boxes:
         if isinstance(item, int):
             return Boxes(self.tensor[item].view(1, -1))
         b = self.tensor[item]
-        assert (
+        assert b.dim() == 2, (
-            b.dim() == 2
+            "Indexing on Boxes with {} failed to return a matrix!".format(item)
-        ), "Indexing on Boxes with {} failed to return a matrix!".format(item)
+        )
         return Boxes(b)
 
     def __len__(self) -> int:
@@ -423,7 +425,7 @@ def matched_pairwise_iou(boxes1: Boxes, boxes2: Boxes) -> torch.Tensor:
         Tensor: iou, sized [N].
     """
     assert len(boxes1) == len(boxes2), (
-        "boxlists should have the same" "number of entries, got {}, {}".format(
+        "boxlists should have the samenumber of entries, got {}, {}".format(
             len(boxes1), len(boxes2)
         )
     )

sam3/agent/helpers/color_map.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 """
 An awesome colormap for really neat visualizations.
 Copied from Detectron, and removed gray colors.

sam3/agent/helpers/keypoints.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 from typing import Any, List, Tuple, Union
 
 import numpy as np

sam3/agent/helpers/mask_overlap_removal.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 from typing import Dict, List
 
 import numpy as np

sam3/agent/helpers/masks.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import copy
 import itertools
 from typing import Any, Iterator, List, Union
@@ -11,7 +13,6 @@ from torch import device
 
 from .boxes import Boxes
 from .memory import retry_if_cuda_oom
-
 from .roi_align import ROIAlign
 
 
@@ -140,10 +141,10 @@ class BitMasks:
         if isinstance(item, int):
             return BitMasks(self.tensor[item].unsqueeze(0))
         m = self.tensor[item]
-        assert (
+        assert m.dim() == 3, (
-            m.dim() == 3
+            "Indexing on BitMasks with {} returns a tensor with shape {}!".format(
-        ), "Indexing on BitMasks with {} returns a tensor with shape {}!".format(
+                item, m.shape
-            item, m.shape
+            )
         )
         return BitMasks(m)
 

sam3/agent/helpers/memory.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import logging
 from contextlib import contextmanager
 from functools import wraps

sam3/agent/helpers/rle.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 """Some utilities for RLE encoding that doesn't require downloading the masks to the cpu"""
 
 import numpy as np

sam3/agent/helpers/roi_align.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 from torch import nn
 from torchvision.ops import roi_align
 

sam3/agent/helpers/rotated_boxes.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import math
@@ -361,9 +363,9 @@ class RotatedBoxes(Boxes):
         if isinstance(item, int):
             return RotatedBoxes(self.tensor[item].view(1, -1))
         b = self.tensor[item]
-        assert (
+        assert b.dim() == 2, (
-            b.dim() == 2
+            "Indexing on RotatedBoxes with {} failed to return a matrix!".format(item)
-        ), "Indexing on RotatedBoxes with {} failed to return a matrix!".format(item)
+        )
         return RotatedBoxes(b)
 
     def __len__(self) -> int:

sam3/agent/helpers/som_utils.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import colorsys
 from dataclasses import dataclass
 from typing import List, Tuple

sam3/agent/helpers/visualizer.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import colorsys
 import logging
 import math
@@ -18,7 +20,6 @@ from matplotlib.backends.backend_agg import FigureCanvasAgg
 from PIL import Image
 
 from .boxes import Boxes, BoxMode
-
 from .color_map import random_color
 from .keypoints import Keypoints
 from .masks import BitMasks, PolygonMasks
@@ -220,9 +221,9 @@ class _PanopticPrediction:
                 empty_ids.append(id)
         if len(empty_ids) == 0:
             return np.zeros(self._seg.shape, dtype=np.uint8)
-        assert (
+        assert len(empty_ids) == 1, (
-            len(empty_ids) == 1
+            ">1 ids corresponds to no labels. This is currently not supported"
-        ), ">1 ids corresponds to no labels. This is currently not supported"
+        )
         return (self._seg != empty_ids[0]).numpy().astype(np.bool)
 
     def semantic_masks(self):

sam3/agent/helpers/zoom_in.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import io
 import math
 

sam3/agent/inference.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import json
 import os
 
@@ -39,7 +41,7 @@ def run_single_image_inference(
         print(f"Output JSON {output_json_path} already exists. Skipping.")
         return
 
-    print(f"{'-'*30} Starting SAM 3 Agent Session... {'-'*30} ")
+    print(f"{'-' * 30} Starting SAM 3 Agent Session... {'-' * 30} ")
     agent_history, final_output_dict, rendered_final_output = agent_inference(
         image_path,
         text_prompt,
@@ -48,7 +50,7 @@ def run_single_image_inference(
         output_dir=output_dir,
         debug=debug,
     )
-    print(f"{'-'*30} End of SAM 3 Agent Session... {'-'*30} ")
+    print(f"{'-' * 30} End of SAM 3 Agent Session... {'-' * 30} ")
 
     final_output_dict["text_prompt"] = text_prompt
     final_output_dict["image_path"] = image_path

sam3/agent/viz.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import cv2
 import numpy as np
 import pycocotools.mask as mask_utils
@@ -71,7 +73,9 @@ def visualize(
         idx = int(zoom_in_index)
         num_masks = len(input_json.get("pred_masks", []))
         if idx < 0 or idx >= num_masks:
-            raise ValueError(f"zoom_in_index {idx} is out of range (0..{num_masks-1}).")
+            raise ValueError(
+                f"zoom_in_index {idx} is out of range (0..{num_masks - 1})."
+            )
 
         # (1) Replicate zoom_in_and_visualize
         object_data = {

sam3/eval/__init__.py

@@ -1 +1,3 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+# pyre-unsafe

sam3/eval/cgf1_eval.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import contextlib
 import copy
 import json
@@ -124,9 +126,9 @@ class COCOCustom(COCO):
         # MODIFICATION: faster and cached subset check
         if not hasattr(self, "img_id_set"):
             self.img_id_set = set(self.getImgIds())
-        assert set(annsImgIds).issubset(
+        assert set(annsImgIds).issubset(self.img_id_set), (
-            self.img_id_set
+            "Results do not correspond to current coco set"
-        ), "Results do not correspond to current coco set"
+        )
         # END MODIFICATION
         if "caption" in anns[0]:
             imgIds = set([img["id"] for img in res.dataset["images"]]) & set(
@@ -299,9 +301,9 @@ class CGF1Eval(COCOeval):
             TP = (match_scores >= thresh).sum()
             FP = len(dt) - TP
             FN = len(gt) - TP
-            assert (
+            assert FP >= 0 and FN >= 0, (
-                FP >= 0 and FN >= 0
+                f"FP: {FP}, FN: {FN}, TP: {TP}, match_scores: {match_scores}, len(dt): {len(dt)}, len(gt): {len(gt)}, ious: {ious}"
-            ), f"FP: {FP}, FN: {FN}, TP: {TP}, match_scores: {match_scores}, len(dt): {len(dt)}, len(gt): {len(gt)}, ious: {ious}"
+            )
             TPs.append(TP)
             FPs.append(FP)
             FNs.append(FN)
@@ -597,9 +599,9 @@ class CGF1Evaluator:
 
         """
         assert len(self.coco_gts) > 0, "No ground truth provided for evaluation."
-        assert len(self.coco_gts) == len(
+        assert len(self.coco_gts) == len(self.coco_evals), (
-            self.coco_evals
+            "Mismatch in number of ground truths and evaluators."
-        ), "Mismatch in number of ground truths and evaluators."
+        )
 
         if self.verbose:
             print(f"Loading predictions from {pred_file}")
@@ -666,17 +668,17 @@ class CGF1Evaluator:
         if len(scorings) == 1:
             return scorings[0]
 
-        assert (
+        assert scorings[0].ndim == 3, (
-            scorings[0].ndim == 3
+            f"Expecting results in [numCats, numAreas, numImgs] format, got {scorings[0].shape}"
-        ), f"Expecting results in [numCats, numAreas, numImgs] format, got {scorings[0].shape}"
+        )
-        assert (
+        assert scorings[0].shape[0] == 1, (
-            scorings[0].shape[0] == 1
+            f"Expecting a single category, got {scorings[0].shape[0]}"
-        ), f"Expecting a single category, got {scorings[0].shape[0]}"
+        )
 
         for scoring in scorings:
-            assert (
+            assert scoring.shape == scorings[0].shape, (
-                scoring.shape == scorings[0].shape
+                f"Shape mismatch: {scoring.shape}, {scorings[0].shape}"
-            ), f"Shape mismatch: {scoring.shape}, {scorings[0].shape}"
+            )
 
         selected_imgs = []
         for img_id in range(scorings[0].shape[-1]):

sam3/eval/coco_eval.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 """
 COCO evaluator that works in distributed mode.
 
@@ -16,19 +18,15 @@ import os
 import pickle
 from collections import defaultdict
 from pathlib import Path
-
 from typing import Any, List, Optional
 
 import numpy as np
-
 import pycocotools.mask as mask_utils
 import torch
 from iopath.common.file_io import g_pathmgr
 from pycocotools.coco import COCO
 from pycocotools.cocoeval import COCOeval
-
 from sam3.train.masks_ops import rle_encode
-
 from sam3.train.utils.distributed import (
     all_gather,
     gather_to_rank_0_via_filesys,
@@ -753,9 +751,9 @@ def loadRes(self, resFile):
         anns = resFile
     assert type(anns) == list, "results in not an array of objects"
     annsImgIds = [ann["image_id"] for ann in anns]
-    assert set(annsImgIds) == (
+    assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), (
-        set(annsImgIds) & set(self.getImgIds())
+        "Results do not correspond to current coco set"
-    ), "Results do not correspond to current coco set"
+    )
     if "caption" in anns[0]:
         imgIds = set([img["id"] for img in res.dataset["images"]]) & set(
             [ann["image_id"] for ann in anns]

sam3/eval/coco_eval_offline.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 """
 This evaluator is meant for regular COCO mAP evaluation, for example on the COCO val set.
 

sam3/eval/coco_reindex.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 """
 Self-contained COCO JSON re-indexing function that creates temporary files.
 """

sam3/eval/coco_writer.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 """
 COCO prediction dumper for distributed training.
 
@@ -81,9 +83,9 @@ class PredictionDumper:
         self.merge_predictions = merge_predictions
         self.pred_file_evaluators = pred_file_evaluators
         if self.pred_file_evaluators is not None:
-            assert (
+            assert merge_predictions, (
-                merge_predictions
+                "merge_predictions must be True if pred_file_evaluators are provided"
-            ), "merge_predictions must be True if pred_file_evaluators are provided"
+            )
         assert self.dump_dir is not None, "dump_dir must be provided"
 
         if is_main_process():

sam3/eval/conversion_util.py

@@ -1,4 +1,6 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+# pyre-unsafe
 import json
 import os
 from collections import defaultdict

sam3/eval/demo_eval.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 """
 This evaluator is based upon COCO evaluation, but evaluates the model in a "demo" setting.
 This means that the model's predictions are thresholded and evaluated as "hard" predictions.
@@ -11,11 +13,9 @@ from typing import Optional
 import numpy as np
 import pycocotools.mask as maskUtils
 from pycocotools.cocoeval import COCOeval
-
 from sam3.eval.coco_eval import CocoEvaluator
 from sam3.train.masks_ops import compute_F_measure
 from sam3.train.utils.distributed import is_main_process
-
 from scipy.optimize import linear_sum_assignment
 
 
@@ -154,9 +154,9 @@ class DemoEval(COCOeval):
             TP = (match_scores >= thresh).sum()
             FP = len(dt) - TP
             FN = len(gt) - TP
-            assert (
+            assert FP >= 0 and FN >= 0, (
-                FP >= 0 and FN >= 0
+                f"FP: {FP}, FN: {FN}, TP: {TP}, match_scores: {match_scores}, len(dt): {len(dt)}, len(gt): {len(gt)}, ious: {ious}"
-            ), f"FP: {FP}, FN: {FN}, TP: {TP}, match_scores: {match_scores}, len(dt): {len(dt)}, len(gt): {len(gt)}, ious: {ious}"
+            )
             TPs.append(TP)
             FPs.append(FP)
             FNs.append(FN)
@@ -526,17 +526,17 @@ class DemoEvaluator(CocoEvaluator):
         if len(scorings) == 1:
             return scorings[0]
 
-        assert (
+        assert scorings[0].ndim == 3, (
-            scorings[0].ndim == 3
+            f"Expecting results in [numCats, numAreas, numImgs] format, got {scorings[0].shape}"
-        ), f"Expecting results in [numCats, numAreas, numImgs] format, got {scorings[0].shape}"
+        )
-        assert (
+        assert scorings[0].shape[0] == 1, (
-            scorings[0].shape[0] == 1
+            f"Expecting a single category, got {scorings[0].shape[0]}"
-        ), f"Expecting a single category, got {scorings[0].shape[0]}"
+        )
 
         for scoring in scorings:
-            assert (
+            assert scoring.shape == scorings[0].shape, (
-                scoring.shape == scorings[0].shape
+                f"Shape mismatch: {scoring.shape}, {scorings[0].shape}"
-            ), f"Shape mismatch: {scoring.shape}, {scorings[0].shape}"
+            )
 
         selected_imgs = []
         for img_id in range(scorings[0].shape[-1]):

sam3/eval/hota_eval_toolkit/__init__.py

@@ -1 +1,3 @@
 # flake8: noqa
+
+# pyre-unsafe

sam3/eval/hota_eval_toolkit/run_ytvis_eval.py

@@ -1,5 +1,7 @@
 # flake8: noqa
 
+# pyre-unsafe
+
 """run_youtube_vis.py
 Run example:
 run_youtube_vis.py --USE_PARALLEL False --METRICS HOTA --TRACKERS_TO_EVAL STEm_Seg

sam3/eval/hota_eval_toolkit/trackeval/__init__.py

@@ -1,4 +1,6 @@
 # flake8: noqa
 
+# pyre-unsafe
+
 from . import datasets, metrics, utils
 from .eval import Evaluator

sam3/eval/hota_eval_toolkit/trackeval/_timing.py

@@ -1,5 +1,7 @@
 # flake8: noqa
 
+# pyre-unsafe
+
 import inspect
 from functools import wraps
 from time import perf_counter

sam3/eval/hota_eval_toolkit/trackeval/datasets/__init__.py

@@ -1,4 +1,6 @@
 # flake8: noqa
 
+# pyre-unsafe
+
 from .tao_ow import TAO_OW
 from .youtube_vis import YouTubeVIS

sam3/eval/hota_eval_toolkit/trackeval/datasets/_base_dataset.py

@@ -1,5 +1,7 @@
 # flake8: noqa
 
+# pyre-unsafe
+
 import csv
 import io
 import os

sam3/eval/hota_eval_toolkit/trackeval/datasets/tao_ow.py

@@ -1,5 +1,7 @@
 # flake8: noqa
 
+# pyre-unsafe
+
 import itertools
 import json
 import os

sam3/eval/hota_eval_toolkit/trackeval/datasets/youtube_vis.py

@@ -1,5 +1,7 @@
 # flake8: noqa
 
+# pyre-unsafe
+
 # note: this file has been modified from its original version in TrackEval in
 # https://github.com/JonathonLuiten/TrackEval/blob/master/trackeval/datasets/youtube_vis.py
 # to support the following:

sam3/eval/hota_eval_toolkit/trackeval/eval.py

@@ -1,5 +1,7 @@
 # flake8: noqa
 
+# pyre-unsafe
+
 import os
 import time
 import traceback
@@ -253,9 +255,10 @@ class Evaluator:
                         if show_progressbar and TQDM_IMPORTED:
                             seq_list_sorted = sorted(seq_list)
 
-                            with Pool(config["NUM_PARALLEL_CORES"]) as pool, tqdm.tqdm(
+                            with (
-                                total=len(seq_list)
+                                Pool(config["NUM_PARALLEL_CORES"]) as pool,
-                            ) as pbar:
+                                tqdm.tqdm(total=len(seq_list)) as pbar,
+                            ):
                                 _eval_sequence = partial(
                                     eval_sequence,
                                     dataset=dataset,

sam3/eval/hota_eval_toolkit/trackeval/metrics/__init__.py

@@ -1,4 +1,6 @@
 # flake8: noqa
 
+# pyre-unsafe
+
 from .count import Count
 from .hota import HOTA

sam3/eval/hota_eval_toolkit/trackeval/metrics/_base_metric.py

@@ -1,5 +1,7 @@
 # flake8: noqa
 
+# pyre-unsafe
+
 from abc import ABC, abstractmethod
 
 import numpy as np

sam3/eval/hota_eval_toolkit/trackeval/metrics/count.py

@@ -1,5 +1,7 @@
 # flake8: noqa
 
+# pyre-unsafe
+
 from .. import _timing
 from ._base_metric import _BaseMetric
 

sam3/eval/hota_eval_toolkit/trackeval/metrics/hota.py

@@ -1,5 +1,7 @@
 # flake8: noqa
 
+# pyre-unsafe
+
 import os
 
 import numpy as np

sam3/eval/hota_eval_toolkit/trackeval/utils.py

@@ -1,5 +1,7 @@
 # flake8: noqa
 
+# pyre-unsafe
+
 import argparse
 import csv
 import os

sam3/eval/postprocessors.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 """Postprocessors class to transform MDETR output according to the downstream task"""
 
 import dataclasses
@@ -81,9 +83,9 @@ class PostProcessImage(nn.Module):
             ret_tensordict: Experimental argument. If true, return a tensordict.TensorDict instead of a list of dictionaries for easier manipulation.
         """
         if ret_tensordict:
-            assert (
+            assert consistent is True, (
-                consistent is True
+                "We don't support returning TensorDict if the outputs have different shapes"
-            ), "We don't support returning TensorDict if the outputs have different shapes"  # NOTE: It's possible but we don't support it.
+            )  # NOTE: It's possible but we don't support it.
             assert self.detection_threshold <= 0.0, "TODO: implement?"
             try:
                 from tensordict import TensorDict
@@ -116,7 +118,9 @@ class PostProcessImage(nn.Module):
 
         if boxes is None:
             assert out_masks is not None
-            assert not ret_tensordict, "We don't support returning TensorDict if the output does not contain boxes"
+            assert not ret_tensordict, (
+                "We don't support returning TensorDict if the output does not contain boxes"
+            )
             B = len(out_masks)
             boxes = [None] * B
             scores = [None] * B
@@ -416,9 +420,9 @@ class PostProcessAPIVideo(PostProcessImage):
             if video_id == -1:
                 video_id = unique_vid_id.item()
             else:
-                assert (
+                assert video_id == unique_vid_id.item(), (
-                    video_id == unique_vid_id.item()
+                    "We can only postprocess one video per datapoint"
-                ), "We can only postprocess one video per datapoint"
+                )
             # keeping track of which objects appear in the current frame
             obj_ids_per_frame = frame_outs["pred_object_ids"]
             assert obj_ids_per_frame.size(-1) == frame_outs["pred_logits"].size(-2)

sam3/eval/saco_veval_eval.py

@@ -1,4 +1,6 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+# pyre-unsafe
 import argparse
 import json
 import os

sam3/eval/saco_veval_evaluators.py

@@ -1,4 +1,6 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+# pyre-unsafe
 import json
 import os
 import tempfile

sam3/eval/teta_eval_toolkit/__init__.py

@@ -1,5 +1,7 @@
 # fmt: off
 # flake8: noqa
 
+# pyre-unsafe
+
 from . import config, datasets, metrics, utils
 from .eval import Evaluator

sam3/eval/teta_eval_toolkit/_timing.py

@@ -1,6 +1,8 @@
 # fmt: off
 # flake8: noqa
 
+# pyre-unsafe
+
 import inspect
 from functools import wraps
 from time import perf_counter

sam3/eval/teta_eval_toolkit/config.py

@@ -1,6 +1,8 @@
 # fmt: off
 # flake8: noqa
 
+# pyre-unsafe
+
 """Config."""
 import argparse
 import os

sam3/eval/teta_eval_toolkit/datasets/__init__.py

@@ -1,5 +1,7 @@
 # fmt: off
 # flake8: noqa
+
+# pyre-unsafe
 """Datasets."""
 from .coco import COCO
 from .tao import TAO

sam3/eval/teta_eval_toolkit/datasets/_base_dataset.py

@@ -1,6 +1,8 @@
 # fmt: off
 # flake8: noqa
 
+# pyre-unsafe
+
 import csv
 import io
 import os

sam3/eval/teta_eval_toolkit/datasets/coco.py

@@ -1,6 +1,8 @@
 # fmt: off
 # flake8: noqa
 
+# pyre-unsafe
+
 """COCO Dataset."""
 import copy
 import itertools

sam3/eval/teta_eval_toolkit/datasets/tao.py

@@ -1,6 +1,8 @@
 # fmt: off
 # flake8: noqa
 
+# pyre-unsafe
+
 """TAO Dataset."""
 import copy
 import itertools

sam3/eval/teta_eval_toolkit/eval.py

@@ -1,6 +1,8 @@
 # fmt: off
 # flake8: noqa
 
+# pyre-unsafe
+
 import copy
 import os
 import pickle

sam3/eval/teta_eval_toolkit/metrics/__init__.py

@@ -1,4 +1,6 @@
 # fmt: off
 # flake8: noqa
 
+# pyre-unsafe
+
 from .teta import TETA

sam3/eval/teta_eval_toolkit/metrics/_base_metric.py

@@ -1,6 +1,8 @@
 # fmt: off
 # flake8: noqa
 
+# pyre-unsafe
+
 from abc import ABC, abstractmethod
 
 import numpy as np

sam3/eval/teta_eval_toolkit/metrics/teta.py

@@ -1,6 +1,8 @@
 # fmt: off
 # flake8: noqa
 
+# pyre-unsafe
+
 """Track Every Thing Accuracy metric."""
 
 import numpy as np

sam3/eval/teta_eval_toolkit/utils.py

@@ -1,6 +1,8 @@
 # fmt: off
 # flake8: noqa
 
+# pyre-unsafe
+
 import csv
 import os
 from collections import OrderedDict

sam3/eval/ytvis_coco_wrapper.py

@@ -1,6 +1,8 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 # (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
 
+# pyre-unsafe
+
 import copy
 import json
 import logging
@@ -93,9 +95,9 @@ class YTVIS(COCO):
             anns = resFile
         assert type(anns) == list, "results is not an array of objects"
         annsImgIds = [ann["image_id"] for ann in anns]
-        assert set(annsImgIds) == (
+        assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), (
-            set(annsImgIds) & set(self.getImgIds())
+            "Results do not correspond to current coco set"
-        ), "Results do not correspond to current coco set"
+        )
         if "bboxes" in anns[0] and not anns[0]["bboxes"] == []:
             res.dataset["categories"] = copy.deepcopy(self.dataset["categories"])
             for id, ann in enumerate(anns):

sam3/eval/ytvis_eval.py

@@ -1,4 +1,6 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+# pyre-unsafe
 import copy
 import gc
 import logging
@@ -107,9 +109,7 @@ class YTVISevalMixin:
             )  # Num preds x Num GTS x Num frames
             inter = inter.sum(-1)
             union = union.sum(-1)
-            assert (
+            assert (union > 0).all(), (
-                union > 0
-            ).all(), (
                 "There exists a tracklet with zero GTs across time. This is suspicious"
             )
             return inter / union
@@ -134,9 +134,9 @@ class YTVISevalMixin:
                 iou = inter / union
                 assert iou >= 0 and iou <= 1, "Encountered an error in IoU computation"
             else:
-                assert np.isclose(inter, 0) and np.isclose(
+                assert np.isclose(inter, 0) and np.isclose(union, 0), (
-                    union, 0
+                    "Encountered an error in IoU computation"
-                ), "Encountered an error in IoU computation"
+                )
                 iou = 1
             return iou
 
@@ -204,16 +204,16 @@ class YTVISResultsWriter:
             if len(prediction) == 0:
                 continue
             for k in ["boxes", "scores", "labels"]:
-                assert (
+                assert k in prediction, (
-                    k in prediction
+                    f"Expected predictions to have `{k}` key, available keys are {prediction.keys()}"
-                ), f"Expected predictions to have `{k}` key, available keys are {prediction.keys()}"
+                )
             if self.save_per_frame_scores:
-                assert (
+                assert "per_frame_scores" in prediction, (
-                    "per_frame_scores" in prediction
+                    f"Expected predictions to have `per_frame_scores` key, available keys are {prediction.keys()}"
-                ), f"Expected predictions to have `per_frame_scores` key, available keys are {prediction.keys()}"
+                )
-            assert xor(
+            assert xor("masks" in prediction, "masks_rle" in prediction), (
-                "masks" in prediction, "masks_rle" in prediction
+                f"Expected predictions to have either `masks` key or `masks_rle` key, available keys are {prediction.keys()}"
-            ), f"Expected predictions to have either `masks` key or `masks_rle` key, available keys are {prediction.keys()}"
+            )
 
             boxes = prediction["boxes"]
             boxes = convert_to_xywh(boxes).tolist()
@@ -221,9 +221,9 @@ class YTVISResultsWriter:
             labels = prediction["labels"].tolist()
             if "masks" in prediction:
                 masks = prediction["masks"].squeeze(2)
-                assert (
+                assert masks.ndim == 4, (
-                    masks.ndim == 4
+                    "Expected masks to be of shape(N_preds,T_frames,H,W)"
-                ), "Expected masks to be of shape(N_preds,T_frames,H,W)"
+                )
 
                 areas = [mask.flatten(1).sum(1).tolist() for mask in masks]
                 rles = [rle_encode(masklet) for masklet in masks]

sam3/logger.py

@@ -1,4 +1,6 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+# pyre-unsafe
 import logging
 import os
 
@@ -40,9 +42,9 @@ def get_logger(name, level=logging.INFO):
     """A command line logger."""
     if "LOG_LEVEL" in os.environ:
         level = os.environ["LOG_LEVEL"].upper()
-        assert (
+        assert level in LOG_LEVELS, (
-            level in LOG_LEVELS
+            f"Invalid LOG_LEVEL: {level}, must be one of {list(LOG_LEVELS.keys())}"
-        ), f"Invalid LOG_LEVEL: {level}, must be one of {list(LOG_LEVELS.keys())}"
+        )
         level = LOG_LEVELS[level]
     logger = logging.getLogger(name)
     logger.setLevel(level)

sam3/model/__init__.py

@@ -1 +1,3 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+# pyre-unsafe

sam3/model/act_ckpt_utils.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import inspect
 from functools import wraps
 from typing import Callable, TypeVar, Union

sam3/model/box_ops.py

@@ -1,4 +1,6 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+# pyre-unsafe
 """
 Utilities for bounding box manipulation and GIoU.
 """

sam3/model/data_misc.py

@@ -1,11 +1,12 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+# pyre-unsafe
 """
 Misc functions, including distributed helpers.
 """
 
 import collections
 import re
-
 from dataclasses import dataclass, field as field_ptr_behaviour, fields, is_dataclass
 from typing import Any, get_args, get_origin, List, Mapping, Optional, Sequence, Union
 
@@ -27,9 +28,9 @@ def interpolate(
             input, size, scale_factor, mode, align_corners
         )
 
-    assert (
+    assert input.shape[0] != 0 or input.shape[1] != 0, (
-        input.shape[0] != 0 or input.shape[1] != 0
+        "At least one of the two first dimensions must be non zero"
-    ), "At least one of the two first dimensions must be non zero"
+    )
 
     if input.shape[1] == 0:
         # Pytorch doesn't support null dimension on the channel dimension, so we transpose to fake a null batch dim

sam3/model/decoder.py

@@ -1,4 +1,6 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+# pyre-unsafe
 """
 Transformer decoder.
 Inspired from Pytorch's version, adds the pre-norm variant
@@ -7,18 +9,13 @@ Inspired from Pytorch's version, adds the pre-norm variant
 from typing import Any, Dict, List, Optional
 
 import numpy as np
-
 import torch
-
 from sam3.sam.transformer import RoPEAttention
-
 from torch import nn, Tensor
 from torchvision.ops.roi_align import RoIAlign
 
 from .act_ckpt_utils import activation_ckpt_wrapper
-
 from .box_ops import box_cxcywh_to_xyxy
-
 from .model_misc import (
     gen_sineembed_for_position,
     get_activation_fn,
@@ -442,9 +439,9 @@ class TransformerDecoder(nn.Module):
             - valid_ratios/spatial_shapes: bs, nlevel, 2
         """
         if memory_mask is not None:
-            assert (
+            assert self.boxRPB == "none", (
-                self.boxRPB == "none"
+                "inputting a memory_mask in the presence of boxRPB is unexpected/not implemented"
-            ), "inputting a memory_mask in the presence of boxRPB is unexpected/not implemented"
+            )
 
         apply_dac = apply_dac if apply_dac is not None else self.dac
         if apply_dac:
@@ -514,18 +511,18 @@ class TransformerDecoder(nn.Module):
             query_pos = self.ref_point_head(query_sine_embed)  # nq, bs, d_model
 
             if self.boxRPB != "none" and reference_boxes is not None:
-                assert (
+                assert spatial_shapes.shape[0] == 1, (
-                    spatial_shapes.shape[0] == 1
+                    "only single scale support implemented"
-                ), "only single scale support implemented"
+                )
                 memory_mask = self._get_rpb_matrix(
                     reference_boxes,
                     (spatial_shapes[0, 0], spatial_shapes[0, 1]),
                 )
                 memory_mask = memory_mask.flatten(0, 1)  # (bs*n_heads, nq, H*W)
             if self.training:
-                assert (
+                assert self.use_act_checkpoint, (
-                    self.use_act_checkpoint
+                    "Activation checkpointing not enabled in the decoder"
-                ), "Activation checkpointing not enabled in the decoder"
+                )
             output, presence_out = activation_ckpt_wrapper(layer)(
                 tgt=output,
                 tgt_query_pos=query_pos,
@@ -674,9 +671,9 @@ class TransformerEncoderCrossAttention(nn.Module):
                 src_pos[0],
             )
 
-        assert (
+        assert src.shape[1] == prompt.shape[1], (
-            src.shape[1] == prompt.shape[1]
+            "Batch size must be the same for src and prompt"
-        ), "Batch size must be the same for src and prompt"
+        )
 
         output = src
 

sam3/model/edt.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 """Triton kernel for euclidean distance transform (EDT)"""
 
 import torch

sam3/model/encoder.py

@@ -1,6 +1,8 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 # Based on https://github.com/IDEA-Research/GroundingDINO
 
+# pyre-unsafe
+
 from typing import Any, Dict, List, Optional, Tuple
 
 import torch
@@ -320,9 +322,9 @@ class TransformerEncoder(nn.Module):
         return reference_points
 
     def _prepare_multilevel_features(self, srcs, masks, pos_embeds):
-        assert (
+        assert len(srcs) == self.num_feature_levels, (
-            len(srcs) == self.num_feature_levels
+            "mismatch between expected and received # of feature levels"
-        ), "mismatch between expected and received # of feature levels"
+        )
 
         src_flatten = []
         mask_flatten = []
@@ -404,9 +406,9 @@ class TransformerEncoder(nn.Module):
             - spatial_shapes: Spatial dimensions of each feature level
             - valid_ratios: Valid ratios for each feature level
         """
-        assert (
+        assert len(src) == self.num_feature_levels, (
-            len(src) == self.num_feature_levels
+            "must be equal to num_feature_levels"
-        ), "must be equal to num_feature_levels"
+        )
         if src_key_padding_masks is not None:
             assert len(src_key_padding_masks) == self.num_feature_levels
         if pos is not None:
@@ -536,9 +538,9 @@ class TransformerEncoderFusion(TransformerEncoder):
                     else None
                 )
         else:
-            assert all(
+            assert all(x.dim == 4 for x in src), (
-                x.dim == 4 for x in src
+                "expected list of (bs, c, h, w) tensors"
-            ), "expected list of (bs, c, h, w) tensors"
+            )
 
         if self.add_pooled_text_to_img_feat:
             # Fusion: Add mean pooled text to image features

sam3/model/geometry_encoders.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 from typing import Tuple
 
 import torch
@@ -9,7 +11,6 @@ from typing_extensions import override
 
 from .act_ckpt_utils import activation_ckpt_wrapper
 from .box_ops import box_cxcywh_to_xyxy
-
 from .model_misc import get_clones
 
 
@@ -146,54 +147,42 @@ class Prompt:
         )
 
         # Dimension checks
-        assert (
+        assert box_embeddings is not None and list(box_embeddings.shape[:2]) == [
-            box_embeddings is not None
+            box_seq_len,
-            and list(box_embeddings.shape[:2])
+            bs,
-            == [
+        ], (
-                box_seq_len,
+            f"Wrong dimension for box embeddings. Expected [{box_seq_len}, {bs}, *] got {box_embeddings.shape}"
-                bs,
+        )
-            ]
+        assert box_mask is not None and list(box_mask.shape) == [
-        ), f"Wrong dimension for box embeddings. Expected [{box_seq_len}, {bs}, *] got {box_embeddings.shape}"
+            bs,
-        assert (
+            box_seq_len,
-            box_mask is not None
+        ], (
-            and list(box_mask.shape)
+            f"Wrong dimension for box mask. Expected [{bs}, {box_seq_len}] got {box_mask.shape}"
-            == [
+        )
-                bs,
+        assert point_embeddings is not None and list(point_embeddings.shape[:2]) == [
-                box_seq_len,
+            point_seq_len,
-            ]
+            bs,
-        ), f"Wrong dimension for box mask. Expected [{bs}, {box_seq_len}] got {box_mask.shape}"
+        ], (
-        assert (
+            f"Wrong dimension for point embeddings. Expected [{point_seq_len}, {bs}, *] got {point_embeddings.shape}"
-            point_embeddings is not None
+        )
-            and list(point_embeddings.shape[:2])
+        assert point_mask is not None and list(point_mask.shape) == [
-            == [
+            bs,
-                point_seq_len,
+            point_seq_len,
-                bs,
+        ], (
-            ]
+            f"Wrong dimension for point mask. Expected [{bs}, {point_seq_len}] got {point_mask.shape}"
-        ), f"Wrong dimension for point embeddings. Expected [{point_seq_len}, {bs}, *] got {point_embeddings.shape}"
+        )
-        assert (
+        assert box_labels is not None and list(box_labels.shape) == [
-            point_mask is not None
+            box_seq_len,
-            and list(point_mask.shape)
+            bs,
-            == [
+        ], (
-                bs,
+            f"Wrong dimension for box labels. Expected [{box_seq_len}, {bs}] got {box_labels.shape}"
-                point_seq_len,
+        )
-            ]
+        assert point_labels is not None and list(point_labels.shape) == [
-        ), f"Wrong dimension for point mask. Expected [{bs}, {point_seq_len}] got {point_mask.shape}"
+            point_seq_len,
-        assert (
+            bs,
-            box_labels is not None
+        ], (
-            and list(box_labels.shape)
+            f"Wrong dimension for point labels. Expected [{point_seq_len}, {bs}] got {point_labels.shape}"
-            == [
+        )
-                box_seq_len,
-                bs,
-            ]
-        ), f"Wrong dimension for box labels. Expected [{box_seq_len}, {bs}] got {box_labels.shape}"
-        assert (
-            point_labels is not None
-            and list(point_labels.shape)
-            == [
-                point_seq_len,
-                bs,
-            ]
-        ), f"Wrong dimension for point labels. Expected [{point_seq_len}, {bs}] got {point_labels.shape}"
         assert (
             # Allowed to be None, we leave it to the encoder to check for validity before encoding.
             mask_embeddings is None
@@ -202,41 +191,41 @@ class Prompt:
                 mask_seq_len,
                 bs,
             ]
-        ), f"Wrong dimension for mask embeddings. Expected [{mask_seq_len}, {bs}, *] got {mask_embeddings.shape}"
+        ), (
-        assert (
+            f"Wrong dimension for mask embeddings. Expected [{mask_seq_len}, {bs}, *] got {mask_embeddings.shape}"
-            mask_mask is None
+        )
-            or list(mask_mask.shape)
+        assert mask_mask is None or list(mask_mask.shape) == [
-            == [
+            bs,
-                bs,
+            mask_seq_len,
-                mask_seq_len,
+        ], (
-            ]
+            f"Wrong dimension for mask attn. mask. Expected [{bs}, {mask_seq_len}] got {mask_mask.shape}"
-        ), f"Wrong dimension for mask attn. mask. Expected [{bs}, {mask_seq_len}] got {mask_mask.shape}"
+        )
 
         # Device checks
-        assert (
+        assert box_embeddings is not None and box_embeddings.device == device, (
-            box_embeddings is not None and box_embeddings.device == device
+            f"Expected box embeddings to be on device {device}, got {box_embeddings.device}"
-        ), f"Expected box embeddings to be on device {device}, got {box_embeddings.device}"
+        )
-        assert (
+        assert box_mask is not None and box_mask.device == device, (
-            box_mask is not None and box_mask.device == device
+            f"Expected box mask to be on device {device}, got {box_mask.device}"
-        ), f"Expected box mask to be on device {device}, got {box_mask.device}"
+        )
-        assert (
+        assert box_labels is not None and box_labels.device == device, (
-            box_labels is not None and box_labels.device == device
+            f"Expected box labels to be on device {device}, got {box_labels.device}"
-        ), f"Expected box labels to be on device {device}, got {box_labels.device}"
+        )
-        assert (
+        assert point_embeddings is not None and point_embeddings.device == device, (
-            point_embeddings is not None and point_embeddings.device == device
+            f"Expected point embeddings to be on device {device}, got {point_embeddings.device}"
-        ), f"Expected point embeddings to be on device {device}, got {point_embeddings.device}"
+        )
-        assert (
+        assert point_mask is not None and point_mask.device == device, (
-            point_mask is not None and point_mask.device == device
+            f"Expected point mask to be on device {device}, got {point_mask.device}"
-        ), f"Expected point mask to be on device {device}, got {point_mask.device}"
+        )
-        assert (
+        assert point_labels is not None and point_labels.device == device, (
-            point_labels is not None and point_labels.device == device
+            f"Expected point labels to be on device {device}, got {point_labels.device}"
-        ), f"Expected point labels to be on device {device}, got {point_labels.device}"
+        )
-        assert (
+        assert mask_embeddings is None or mask_embeddings.device == device, (
-            mask_embeddings is None or mask_embeddings.device == device
+            f"Expected mask embeddings to be on device {device}, got {mask_embeddings.device}"
-        ), f"Expected mask embeddings to be on device {device}, got {mask_embeddings.device}"
+        )
-        assert (
+        assert mask_mask is None or mask_mask.device == device, (
-            mask_mask is None or mask_mask.device == device
+            f"Expected mask attn. mask to be on device {device}, got {mask_mask.device}"
-        ), f"Expected mask attn. mask to be on device {device}, got {mask_mask.device}"
+        )
 
         self.box_embeddings = box_embeddings
         self.point_embeddings = point_embeddings
@@ -262,30 +251,30 @@ class Prompt:
         if point_embeddings is not None:
             point_seq_len = point_embeddings.shape[0]
             if bs is not None:
-                assert (
+                assert bs == point_embeddings.shape[1], (
-                    bs == point_embeddings.shape[1]
+                    f"Batch size mismatch between box and point embeddings. Got {bs} and {point_embeddings.shape[1]}."
-                ), f"Batch size mismatch between box and point embeddings. Got {bs} and {point_embeddings.shape[1]}."
+                )
             else:
                 bs = point_embeddings.shape[1]
             if device is not None:
-                assert (
+                assert device == point_embeddings.device, (
-                    device == point_embeddings.device
+                    "Device mismatch between box and point embeddings"
-                ), "Device mismatch between box and point embeddings"
+                )
             else:
                 device = point_embeddings.device
 
         if mask_embeddings is not None:
             mask_seq_len = mask_embeddings.shape[0]
             if bs is not None:
-                assert (
+                assert bs == mask_embeddings.shape[1], (
-                    bs == mask_embeddings.shape[1]
+                    f"Batch size mismatch between box/point and mask embedding. Got {bs} and {mask_embeddings.shape[1]}"
-                ), f"Batch size mismatch between box/point and mask embedding. Got {bs} and {mask_embeddings.shape[1]}"
+                )
             else:
                 bs = mask_embeddings.shape[1]
             if device is not None:
-                assert (
+                assert device == mask_embeddings.device, (
-                    device == mask_embeddings.device
+                    "Device mismatch between box/point and mask embeddings."
-                ), "Device mismatch between box/point and mask embeddings."
+                )
             else:
                 device = mask_embeddings.device
 
@@ -537,9 +526,9 @@ class SequenceGeometryEncoder(nn.Module):
         if add_cls:
             self.cls_embed = torch.nn.Embedding(1, self.d_model)
 
-        assert (
+        assert points_direct_project or points_pos_enc or points_pool, (
-            points_direct_project or points_pos_enc or points_pool
+            "Error: need at least one way to encode points"
-        ), "Error: need at least one way to encode points"
+        )
         assert (
             encode_boxes_as_points
             or boxes_direct_project
@@ -581,16 +570,16 @@ class SequenceGeometryEncoder(nn.Module):
 
         self.encode = None
         if num_layers > 0:
-            assert (
+            assert add_cls, (
-                add_cls
+                "It's currently highly recommended to add a CLS when using a transformer"
-            ), "It's currently highly recommended to add a CLS when using a transformer"
+            )
             self.encode = get_clones(layer, num_layers)
             self.encode_norm = nn.LayerNorm(self.d_model)
 
         if mask_encoder is not None:
-            assert isinstance(
+            assert isinstance(mask_encoder, MaskEncoder), (
-                mask_encoder, MaskEncoder
+                f"Expected mask_encoder of type MaskEncoder. Got {type(mask_encoder)}."
-            ), f"Expected mask_encoder of type MaskEncoder. Got {type(mask_encoder)}."
+            )
             if add_mask_label:
                 self.mask_label_embed = torch.nn.Embedding(2, self.d_model)
         self.add_mask_label = add_mask_label
@@ -699,16 +688,15 @@ class SequenceGeometryEncoder(nn.Module):
         img_feats: torch.Tensor = None,
     ):
         n_masks, bs = masks.shape[:2]
-        assert (
+        assert n_masks == 1, (
-            n_masks == 1
+            "We assume one mask per prompt for now. Code should still be functional if this assertion is removed."
-        ), "We assume one mask per prompt for now. Code should still be functional if this assertion is removed."
+        )
-        assert (
+        assert list(attn_mask.shape) == [
-            list(attn_mask.shape)
+            bs,
-            == [
+            n_masks,
-                bs,
+        ], (
-                n_masks,
+            f"Expected attn_mask to be of shape {bs}x{n_masks}. Got {list(attn_mask.shape)}."
-            ]
+        )
-        ), f"Expected attn_mask to be of shape {bs}x{n_masks}. Got {list(attn_mask.shape)}."
         masks, pos = self.mask_encoder(
             masks=masks.flatten(0, 1).float(),
             pix_feat=img_feats,

sam3/model/io_utils.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import contextlib
 import os
 import queue
@@ -11,9 +13,7 @@ import numpy as np
 import torch
 import torch.nn.functional as F
 import torchvision.transforms.functional as TF
-
 from PIL import Image
-
 from sam3.logger import get_logger
 from tqdm import tqdm
 

sam3/model/maskformer_segmentation.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import math
 from typing import Dict, List, Optional
 
@@ -246,7 +248,9 @@ class UniversalSegmentationHead(SegmentationHead):
         self.d_model = hidden_dim
 
         if dot_product_scorer is not None:
-            assert presence_head, "Specifying a dot product scorer without a presence head is likely a mistake"
+            assert presence_head, (
+                "Specifying a dot product scorer without a presence head is likely a mistake"
+            )
 
         self.presence_head = None
         if presence_head:

sam3/model/memory.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import math
 from typing import Tuple
 
@@ -60,9 +62,9 @@ class SimpleMaskDownSampler(nn.Module):
         self.encoder.append(nn.Conv2d(mask_out_chans, embed_dim, kernel_size=1))
         self.interpol_size = interpol_size
         if self.interpol_size is not None:
-            assert isinstance(
+            assert isinstance(self.interpol_size, (list, tuple)), (
-                self.interpol_size, (list, tuple)
+                f"Unsupported type {type(self.interpol_size)}. Should be a list or tuple."
-            ), f"Unsupported type {type(self.interpol_size)}. Should be a list or tuple."
+            )
             self.interpol_size = list(interpol_size)
             assert len(self.interpol_size) == 2
 

sam3/model/model_misc.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 """Various utility models"""
 
 import copy
@@ -328,9 +330,9 @@ class SAM3Output(list):
             self.output = output
         else:
             self.output = []
-        assert isinstance(
+        assert isinstance(iter_mode, SAM3Output.IterMode), (
-            iter_mode, SAM3Output.IterMode
+            f"iter_mode shoulf be of enum type 'SAM3Output.IterMode'. Got {type(iter_mode)}"
-        ), f"iter_mode shoulf be of enum type 'SAM3Output.IterMode'. Got {type(iter_mode)}"
+        )
 
         self.iter_mode = iter_mode
         # We create a weak reference to self to be used in the lambda functions.
@@ -409,9 +411,9 @@ class SAM3Output(list):
         return SAM3Output._IterationMode(model_output=model_output, iter_mode=iter_mode)
 
     def append(self, item: list):
-        assert isinstance(
+        assert isinstance(item, list), (
-            item, list
+            f"Only list items are supported. Got {type(item)}"
-        ), f"Only list items are supported. Got {type(item)}"
+        )
         self.output.append(item)
 
     def __repr__(self):

sam3/model/necks.py

@@ -1,12 +1,13 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 """Necks are the interface between a vision backbone and the rest of the detection model"""
 
 from copy import deepcopy
 from typing import List, Optional, Tuple
 
 import torch
-
 import torch.nn as nn
 
 

sam3/model/position_encoding.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import math
 from typing import Optional
 

sam3/model/sam1_task_predictor.py

@@ -1,19 +1,18 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 # All rights reserved.
 
+# pyre-unsafe
+
 # This source code is licensed under the license found in the
 # LICENSE file in the root directory of this source tree.
 
 import logging
-
 from typing import List, Optional, Tuple, Union
 
 import numpy as np
 import torch
-
 import torch.nn as nn
 from PIL.Image import Image
-
 from sam3.model.sam3_tracker_base import Sam3TrackerBase
 from sam3.model.utils.sam1_utils import SAM2Transforms
 
@@ -95,9 +94,9 @@ class SAM3InteractiveImagePredictor(nn.Module):
         input_image = self._transforms(image)
         input_image = input_image[None, ...].to(self.device)
 
-        assert (
+        assert len(input_image.shape) == 4 and input_image.shape[1] == 3, (
-            len(input_image.shape) == 4 and input_image.shape[1] == 3
+            f"input_image must be of size 1x3xHxW, got {input_image.shape}"
-        ), f"input_image must be of size 1x3xHxW, got {input_image.shape}"
+        )
         logging.info("Computing image embeddings for the provided image...")
         backbone_out = self.model.forward_image(input_image)
         (
@@ -134,17 +133,17 @@ class SAM3InteractiveImagePredictor(nn.Module):
         assert isinstance(image_list, list)
         self._orig_hw = []
         for image in image_list:
-            assert isinstance(
+            assert isinstance(image, np.ndarray), (
-                image, np.ndarray
+                "Images are expected to be an np.ndarray in RGB format, and of shape  HWC"
-            ), "Images are expected to be an np.ndarray in RGB format, and of shape  HWC"
+            )
             self._orig_hw.append(image.shape[:2])
         # Transform the image to the form expected by the model
         img_batch = self._transforms.forward_batch(image_list)
         img_batch = img_batch.to(self.device)
         batch_size = img_batch.shape[0]
-        assert (
+        assert len(img_batch.shape) == 4 and img_batch.shape[1] == 3, (
-            len(img_batch.shape) == 4 and img_batch.shape[1] == 3
+            f"img_batch must be of size Bx3xHxW, got {img_batch.shape}"
-        ), f"img_batch must be of size Bx3xHxW, got {img_batch.shape}"
+        )
         logging.info("Computing image embeddings for the provided images...")
         backbone_out = self.model.forward_image(img_batch)
         (
@@ -300,9 +299,9 @@ class SAM3InteractiveImagePredictor(nn.Module):
     ):
         unnorm_coords, labels, unnorm_box, mask_input = None, None, None, None
         if point_coords is not None:
-            assert (
+            assert point_labels is not None, (
-                point_labels is not None
+                "point_labels must be supplied if point_coords is supplied."
-            ), "point_labels must be supplied if point_coords is supplied."
+            )
             point_coords = torch.as_tensor(
                 point_coords, dtype=torch.float, device=self.device
             )
@@ -439,9 +438,9 @@ class SAM3InteractiveImagePredictor(nn.Module):
             raise RuntimeError(
                 "An image must be set with .set_image(...) to generate an embedding."
             )
-        assert (
+        assert self._features is not None, (
-            self._features is not None
+            "Features must exist if an image has been set."
-        ), "Features must exist if an image has been set."
+        )
         return self._features["image_embed"]
 
     @property

sam3/model/sam3_image.py

@@ -1,24 +1,21 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import os
 from copy import deepcopy
 from typing import Dict, List, Optional, Tuple
 
 import numpy as np
 import torch
-
 from sam3.model.model_misc import SAM3Output
-
 from sam3.model.sam1_task_predictor import SAM3InteractiveImagePredictor
 from sam3.model.vl_combiner import SAM3VLBackbone
 from sam3.perflib.nms import nms_masks
-
 from sam3.train.data.collator import BatchedDatapoint
 
 from .act_ckpt_utils import activation_ckpt_wrapper
-
 from .box_ops import box_cxcywh_to_xyxy
-
 from .geometry_encoders import Prompt
 from .model_misc import inverse_sigmoid
 
@@ -659,9 +656,9 @@ class Sam3Image(torch.nn.Module):
             inference_state["original_heights"],
             inference_state["original_widths"],
         )
-        assert (
+        assert batch_size == len(orig_heights) == len(orig_widths), (
-            batch_size == len(orig_heights) == len(orig_widths)
+            f"Batch size mismatch in predict_inst_batch. Got {batch_size}, {len(orig_heights)}, {len(orig_widths)}"
-        ), f"Batch size mismatch in predict_inst_batch. Got {batch_size}, {len(orig_heights)}, {len(orig_widths)}"
+        )
         feats = [
             feat.permute(1, 2, 0).view(batch_size, -1, *feat_size)
             for feat, feat_size in zip(

sam3/model/sam3_image_processor.py

@@ -1,12 +1,12 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+# pyre-unsafe
 from typing import Dict, List
 
 import numpy as np
 import PIL
 import torch
-
 from sam3.model import box_ops
-
 from sam3.model.data_misc import FindStage, interpolate
 from torchvision.transforms import v2
 
@@ -81,9 +81,9 @@ class Sam3Processor:
         if not isinstance(images, list):
             raise ValueError("Images must be a list of PIL images or tensors")
         assert len(images) > 0, "Images list must not be empty"
-        assert isinstance(
+        assert isinstance(images[0], PIL.Image.Image), (
-            images[0], PIL.Image.Image
+            "Images must be a list of PIL images"
-        ), "Images must be a list of PIL images"
+        )
 
         state["original_heights"] = [image.height for image in images]
         state["original_widths"] = [image.width for image in images]

sam3/model/sam3_tracker_base.py

@@ -1,14 +1,13 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import logging
 
 import torch
 import torch.nn.functional as F
-
 from sam3.model.memory import SimpleMaskEncoder
-
 from sam3.model.sam3_tracker_utils import get_1d_sine_pe, select_closest_cond_frames
-
 from sam3.sam.mask_decoder import MaskDecoder, MLP
 from sam3.sam.prompt_encoder import PromptEncoder
 from sam3.sam.transformer import TwoWayTransformer
@@ -900,8 +899,6 @@ class Sam3TrackerBase(torch.nn.Module):
                 image=current_image,
                 point_inputs=backbone_out["point_inputs_per_frame"].get(stage_id, None),
                 mask_inputs=backbone_out["mask_inputs_per_frame"].get(stage_id, None),
-                gt_masks=backbone_out["gt_masks_per_frame"].get(stage_id, None),
-                frames_to_add_correction_pt=frames_to_add_correction_pt,
                 output_dict=output_dict,
                 num_frames=num_frames,
             )

sam3/model/sam3_tracker_utils.py

@@ -1,10 +1,11 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import numpy as np
 import torch
 import torch.nn.functional as F
 from numpy.typing import NDArray
-
 from sam3.model.edt import edt_triton
 
 

sam3/model/sam3_tracking_predictor.py

@@ -1,10 +1,11 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import logging
 from collections import OrderedDict
 
 import torch
-
 from sam3.model.sam3_tracker_base import concat_points, NO_OBJ_SCORE, Sam3TrackerBase
 from sam3.model.sam3_tracker_utils import fill_holes_in_mask_scores
 from sam3.model.utils.sam2_utils import load_video_frames
@@ -657,8 +658,6 @@ class Sam3TrackerPredictor(Sam3TrackerBase):
             image=image,
             point_inputs=None,
             mask_inputs=mask_inputs,
-            gt_masks=None,
-            frames_to_add_correction_pt=[],
             output_dict={
                 "cond_frame_outputs": {},
                 "non_cond_frame_outputs": {},

sam3/model/sam3_video_base.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import datetime
 import logging
 import math
@@ -14,7 +16,6 @@ import numpy.typing as npt
 import torch
 import torch.distributed as dist
 import torch.nn.functional as F
-
 from sam3 import perflib
 from sam3.logger import get_logger
 from sam3.model.box_ops import fast_diag_box_iou
@@ -618,9 +619,9 @@ class Sam3VideoBase(nn.Module):
                 num_obj_dropped_due_to_limit,
                 trk_id_to_max_iou_high_conf_det,
             ]
-            assert (
+            assert len(update_plan) == NUM_BROADCAST_ITEMS, (
-                len(update_plan) == NUM_BROADCAST_ITEMS
+                f"Manually update NUM_BROADCAST_ITEMS to be: {len(update_plan)}"
-            ), f"Manually update NUM_BROADCAST_ITEMS to be: {len(update_plan)}"
+            )
             self.broadcast_python_obj_cpu(update_plan, src=0)
         elif self.rank > 0 and self.world_size > 1:
             update_plan = [
@@ -840,9 +841,9 @@ class Sam3VideoBase(nn.Module):
         binary_tracker_low_res_masks_global = tracker_low_res_masks_global > 0
         batch_size = tracker_low_res_masks_global.size(0)
         if batch_size > 0:
-            assert (
+            assert len(obj_ids_global) == batch_size, (
-                len(obj_ids_global) == batch_size
+                f"Mismatch in number of objects: {len(obj_ids_global)} vs {batch_size}"
-            ), f"Mismatch in number of objects: {len(obj_ids_global)} vs {batch_size}"
+            )
             NEVER_OCCLUDED = -1
             ALWAYS_OCCLUDED = 100000  # This value should be larger than any possible frame index, indicates that the object was removed by hotstart logic
             last_occluded_prev = torch.cat(
@@ -1021,9 +1022,9 @@ class Sam3VideoBase(nn.Module):
         reverse: bool = False,
     ):
         # Suppress overlapping masks for objects that were most recently occluded
-        assert (
+        assert binary_low_res_masks.dtype == torch.bool, (
-            binary_low_res_masks.dtype == torch.bool
+            f"Expected boolean tensor, got {binary_low_res_masks.dtype}"
-        ), f"Expected boolean tensor, got {binary_low_res_masks.dtype}"
+        )
         to_suppress = torch.zeros(
             binary_low_res_masks.size(0),
             device=binary_low_res_masks.device,
@@ -1128,9 +1129,9 @@ class Sam3VideoBase(nn.Module):
                 num_frames_propagated += 1
 
             # only 1 frames should be propagated
-            assert (
+            assert num_frames_propagated == 1 and out_frame_idx == frame_idx, (
-                num_frames_propagated == 1 and out_frame_idx == frame_idx
+                f"num_frames_propagated: {num_frames_propagated}, out_frame_idx: {out_frame_idx}, frame_idx: {frame_idx}"
-            ), f"num_frames_propagated: {num_frames_propagated}, out_frame_idx: {out_frame_idx}, frame_idx: {frame_idx}"
+            )
             assert isinstance(out_obj_ids, list)
             obj_ids_local.extend(out_obj_ids)
             low_res_masks_list.append(out_low_res_masks.squeeze(1))
@@ -1187,9 +1188,9 @@ class Sam3VideoBase(nn.Module):
 
         assert det_masks.is_floating_point(), "float tensor expected (do not binarize)"
         assert trk_masks.is_floating_point(), "float tensor expected (do not binarize)"
-        assert (
+        assert trk_masks.size(0) == len(trk_obj_ids), (
-            trk_masks.size(0) == len(trk_obj_ids)
+            f"trk_masks and trk_obj_ids should have the same length, {trk_masks.size(0)} vs {len(trk_obj_ids)}"
-        ), f"trk_masks and trk_obj_ids should have the same length, {trk_masks.size(0)} vs {len(trk_obj_ids)}"
+        )
         if trk_masks.size(0) == 0:
             # all detections are new
             new_det_fa_inds = np.arange(det_masks.size(0))
@@ -1653,9 +1654,9 @@ class Sam3VideoBase(nn.Module):
         # a) first, expand "confirmation_data" to include new masklets added in this frame
         status_prev = confirmation_data["status"]
         consecutive_det_num_prev = confirmation_data["consecutive_det_num"]
-        assert (
+        assert status_prev.shape == obj_ids_all_gpu_prev.shape, (
-            status_prev.shape == obj_ids_all_gpu_prev.shape
+            f"Got {status_prev.shape} vs {obj_ids_all_gpu_prev.shape}"
-        ), f"Got {status_prev.shape} vs {obj_ids_all_gpu_prev.shape}"
+        )
 
         obj_id_to_updated_idx = {
             obj_id: idx for idx, obj_id in enumerate(obj_ids_all_gpu_updated)

sam3/model/sam3_video_inference.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import logging
 from collections import defaultdict
 
@@ -7,7 +9,6 @@ import numpy as np
 import torch
 import torch.distributed as dist
 import torch.nn.functional as F
-
 from sam3 import perflib
 from sam3.logger import get_logger
 from sam3.model.act_ckpt_utils import clone_output_wrapper
@@ -553,7 +554,9 @@ class Sam3VideoInference(Sam3VideoBase):
         assert (
             "cached_frame_outputs" in inference_state
             and frame_idx in inference_state["cached_frame_outputs"]
-        ), "No cached outputs found. Ensure normal propagation has run first to populate the cache."
+        ), (
+            "No cached outputs found. Ensure normal propagation has run first to populate the cache."
+        )
         cached_outputs = inference_state["cached_frame_outputs"][frame_idx]
 
         obj_id_to_mask = cached_outputs.copy()
@@ -561,9 +564,9 @@ class Sam3VideoInference(Sam3VideoBase):
         # Update with refined masks if provided
         if refined_obj_id_to_mask is not None:
             for obj_id, refined_mask in refined_obj_id_to_mask.items():
-                assert (
+                assert refined_mask is not None, (
-                    refined_mask is not None
+                    f"Refined mask data must be provided for obj_id {obj_id}"
-                ), f"Refined mask data must be provided for obj_id {obj_id}"
+                )
                 obj_id_to_mask[obj_id] = refined_mask
 
         return obj_id_to_mask
@@ -658,12 +661,12 @@ class Sam3VideoInference(Sam3VideoBase):
         for i, thresh in enumerate(new_det_score_thresh_list):
             self.new_det_thresh = thresh
             for num_objects in num_objects_list:
-                logger.info(f"{i+1}/{num_rounds} warming up model compilation")
+                logger.info(f"{i + 1}/{num_rounds} warming up model compilation")
                 self.add_prompt(
                     inference_state, frame_idx=start_frame_idx, text_str="cat"
                 )
                 logger.info(
-                    f"{i+1}/{num_rounds} warming up model compilation -- simulating {num_objects}/{self.num_obj_for_compile} objects"
+                    f"{i + 1}/{num_rounds} warming up model compilation -- simulating {num_objects}/{self.num_obj_for_compile} objects"
                 )
                 inference_state = self.add_fake_objects_to_inference_state(
                     inference_state, num_objects, frame_idx=start_frame_idx
@@ -688,7 +691,7 @@ class Sam3VideoInference(Sam3VideoBase):
                     pass
                 self.reset_state(inference_state)
                 logger.info(
-                    f"{i+1}/{num_rounds} warming up model compilation -- completed round {i+1} out of {num_rounds}"
+                    f"{i + 1}/{num_rounds} warming up model compilation -- completed round {i + 1} out of {num_rounds}"
                 )
 
         # Warm up Tracker memory encoder with varying input shapes
@@ -852,12 +855,12 @@ class Sam3VideoInference(Sam3VideoBase):
         logger.debug("Running add_prompt on frame %d", frame_idx)
 
         num_frames = inference_state["num_frames"]
-        assert (
+        assert text_str is not None or boxes_xywh is not None, (
-            text_str is not None or boxes_xywh is not None
+            "at least one type of prompt (text, boxes) must be provided"
-        ), "at least one type of prompt (text, boxes) must be provided"
+        )
-        assert (
+        assert 0 <= frame_idx < num_frames, (
-            0 <= frame_idx < num_frames
+            f"{frame_idx=} is out of range for a total of {num_frames} frames"
-        ), f"{frame_idx=} is out of range for a total of {num_frames} frames"
+        )
 
         # since it's a semantic prompt, we start over
         self.reset_state(inference_state)
@@ -1198,9 +1201,9 @@ class Sam3VideoInferenceWithInstanceInteractivity(Sam3VideoInference):
             "propagation_partial",
             "propagation_fetch",
         ]
-        assert (
+        assert action_type in instance_actions + propagation_actions, (
-            action_type in instance_actions + propagation_actions
+            f"Invalid action type: {action_type}, must be one of {instance_actions + propagation_actions}"
-        ), f"Invalid action type: {action_type}, must be one of {instance_actions + propagation_actions}"
+        )
         action = {
             "type": action_type,
             "frame_idx": frame_idx,
@@ -1368,12 +1371,12 @@ class Sam3VideoInferenceWithInstanceInteractivity(Sam3VideoInference):
     ):
         if points is not None:
             # Tracker instance prompts
-            assert (
+            assert text_str is None and boxes_xywh is None, (
-                text_str is None and boxes_xywh is None
+                "When points are provided, text_str and boxes_xywh must be None."
-            ), "When points are provided, text_str and boxes_xywh must be None."
+            )
-            assert (
+            assert obj_id is not None, (
-                obj_id is not None
+                "When points are provided, obj_id must be provided."
-            ), "When points are provided, obj_id must be provided."
+            )
             return self.add_tracker_new_points(
                 inference_state,
                 frame_idx,
@@ -1489,9 +1492,9 @@ class Sam3VideoInferenceWithInstanceInteractivity(Sam3VideoInference):
                 tracker_states = self._get_tracker_inference_states_by_obj_ids(
                     inference_state, [obj_id]
                 )
-                assert (
+                assert len(tracker_states) == 1, (
-                    len(tracker_states) == 1
+                    f"[rank={self.rank}] Multiple Tracker inference states found for the same object id."
-                ), f"[rank={self.rank}] Multiple Tracker inference states found for the same object id."
+                )
                 tracker_state = tracker_states[0]
 
             # log

sam3/model/sam3_video_predictor.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import datetime
 import gc
 import multiprocessing as mp
@@ -14,7 +16,6 @@ from typing import List, Optional
 
 import psutil
 import torch
-
 from sam3.logger import get_logger
 
 logger = get_logger(__name__)
@@ -168,7 +169,7 @@ class Sam3VideoPredictor:
     ):
         """Remove an object from tracking."""
         logger.debug(
-            f"remove object {obj_id} in session {session_id}: " f"{is_user_action=}"
+            f"remove object {obj_id} in session {session_id}: {is_user_action=}"
         )
         session = self._get_session(session_id)
         inference_state = session["state"]

sam3/model/text_encoder_ve.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 from collections import OrderedDict
 from typing import Callable, List, Optional, Tuple, Union
 
@@ -316,9 +318,9 @@ class VETextEncoder(nn.Module):
             # The text is already encoded, use as is.
             text_attention_mask, text_memory_resized, tokenized = text
             inputs_embeds = tokenized["inputs_embeds"]
-            assert (
+            assert input_boxes is None or len(input_boxes) == 0, (
-                input_boxes is None or len(input_boxes) == 0
+                "Can't replace boxes in text if it's already encoded"
-            ), "Can't replace boxes in text if it's already encoded"
+            )
 
         # Note that the input_embeds are returned in pytorch's convention (sequence first)
         return (

sam3/model/tokenizer_ve.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 """
 Text Tokenizer.
 

sam3/model/utils/__init__.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 # All rights reserved.
 
+# pyre-unsafe
+
 # This source code is licensed under the license found in the
 # LICENSE file in the root directory of this source tree.

sam3/model/utils/misc.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 from collections import defaultdict
 from dataclasses import fields, is_dataclass
 from typing import Any, Mapping, Protocol, runtime_checkable

sam3/model/utils/sam1_utils.py

@@ -1,6 +1,8 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 # All rights reserved.
 
+# pyre-unsafe
+
 # This source code is licensed under the license found in the
 # LICENSE file in the root directory of this source tree.
 

sam3/model/utils/sam2_utils.py

@@ -1,6 +1,8 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 # All rights reserved.
 
+# pyre-unsafe
+
 # This source code is licensed under the license found in the
 # LICENSE file in the root directory of this source tree.
 

sam3/model/vitdet.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 """
 ViTDet backbone adapted from Detectron2.
 This module implements Vision Transformer (ViT) backbone for object detection.
@@ -706,9 +708,9 @@ class ViT(nn.Module):
         self.retain_cls_token = retain_cls_token
         if self.retain_cls_token:
             assert pretrain_use_cls_token
-            assert (
+            assert len(window_block_indexes) == 0, (
-                len(window_block_indexes) == 0
+                "windowing not supported with cls token"
-            ), "windowing not supported with cls token"
+            )
 
             assert sum(self.rel_pos_blocks) == 0, "rel pos not supported with cls token"
 

sam3/model/vl_combiner.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 """Provides utility to combine a vision backbone with a language backbone."""
 
 from copy import copy
@@ -7,7 +9,6 @@ from typing import List, Optional
 
 import torch
 import torch.nn as nn
-
 from torch.nn.attention import sdpa_kernel, SDPBackend
 
 from .act_ckpt_utils import activation_ckpt_wrapper

sam3/model_builder.py

@@ -1,8 +1,11 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import os
 from typing import Optional
 
+import pkg_resources
 import torch
 import torch.nn as nn
 from huggingface_hub import hf_hub_download
@@ -558,8 +561,8 @@ def build_sam3_image_model(
     bpe_path=None,
     device="cuda" if torch.cuda.is_available() else "cpu",
     eval_mode=True,
-    checkpoint_path=None,
+    checkpoint_path="/home/quant/data/dev/sam3/sam3.pt",
-    load_from_HF=True,
+    load_from_HF=False,
     enable_segmentation=True,
     enable_inst_interactivity=False,
     compile=False,
@@ -580,9 +583,10 @@ def build_sam3_image_model(
         A SAM3 image model
     """
     if bpe_path is None:
-        bpe_path = os.path.join(
+        bpe_path = pkg_resources.resource_filename(
-            os.path.dirname(__file__), "..", "assets", "bpe_simple_vocab_16e6.txt.gz"
+            "sam3", "assets/bpe_simple_vocab_16e6.txt.gz"
         )
+
     # Create visual components
     compile_mode = "default" if compile else None
     vision_encoder = _create_vision_backbone(
@@ -647,8 +651,8 @@ def download_ckpt_from_hf():
 
 
 def build_sam3_video_model(
-    checkpoint_path: Optional[str] = None,
+    checkpoint_path: Optional[str] = "/home/quant/data/dev/sam3/sam3.pt",
-    load_from_HF=True,
+    load_from_HF=False,
     bpe_path: Optional[str] = None,
     has_presence_token: bool = True,
     geo_encoder_use_img_cross_attn: bool = True,
@@ -668,8 +672,8 @@ def build_sam3_video_model(
         Sam3VideoInferenceWithInstanceInteractivity: The instantiated dense tracking model
     """
     if bpe_path is None:
-        bpe_path = os.path.join(
+        bpe_path = pkg_resources.resource_filename(
-            os.path.dirname(__file__), "..", "assets", "bpe_simple_vocab_16e6.txt.gz"
+            "sam3", "assets/bpe_simple_vocab_16e6.txt.gz"
         )
 
     # Build Tracker module

sam3/perflib/__init__.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import os
 
 is_enabled = False

sam3/perflib/associate_det_trk.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 from collections import defaultdict
 
 import torch

sam3/perflib/compile.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import torch
 
 

sam3/perflib/connected_components.py

@@ -1,4 +1,6 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
+
+# pyre-unsafe
 import logging
 
 import torch
@@ -34,9 +36,9 @@ def connected_components_cpu(input_tensor: torch.Tensor):
     if input_tensor.dim() == 4 and input_tensor.shape[1] == 1:
         input_tensor = input_tensor.squeeze(1)
     else:
-        assert (
+        assert input_tensor.dim() == 3, (
-            input_tensor.dim() == 3
+            "Input tensor must be (B, H, W) or (B, 1, H, W)."
-        ), "Input tensor must be (B, H, W) or (B, 1, H, W)."
+        )
 
     batch_size = input_tensor.shape[0]
     labels_list = []
@@ -65,9 +67,9 @@ def connected_components(input_tensor: torch.Tensor):
     if input_tensor.dim() == 3:
         input_tensor = input_tensor.unsqueeze(1)
 
-    assert (
+    assert input_tensor.dim() == 4 and input_tensor.shape[1] == 1, (
-        input_tensor.dim() == 4 and input_tensor.shape[1] == 1
+        "Input tensor must be (B, H, W) or (B, 1, H, W)."
-    ), "Input tensor must be (B, H, W) or (B, 1, H, W)."
+    )
 
     if input_tensor.is_cuda:
         if HAS_CC_TORCH:

sam3/perflib/fa3.py

@@ -1,5 +1,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
 
+# pyre-unsafe
+
 import torch