Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
fde518c
Add ask_vlm method for cloud VLM alert verification
Jun 18, 2026
9a5e3e1
Automatically reformatting code
Jun 18, 2026
d3a428b
Send ask_vlm query/model_id as form fields; use friendly model alias
Jun 22, 2026
2b20fce
Update ask_vlm model_id docstring examples to vision-capable aliases
Jun 22, 2026
320887b
ask_vlm: rename images -> media, accept up to 8
Jun 22, 2026
00789e0
ask_vlm: point at renamed /v1/vlm-verifications endpoint
Jun 24, 2026
263808d
fix: correct URL path separator and add regression test
Jun 24, 2026
3cfbb7e
address PR comments: model list in docstring, timeout/corrupted-image…
Jun 24, 2026
7216313
trim ask_vlm tests to meaningful coverage only
Jun 24, 2026
6aad9e0
fix CI failures: numpy optional import, magic value constant, pylint …
Jun 25, 2026
6d8b680
address Copilot review comments on ask_vlm
Jun 25, 2026
397f9cc
fix: clarify media docstring and add /v1 deduplication regression test
Jun 25, 2026
2ca16ff
fix: register ask_vlm in CLI command groups
Jun 25, 2026
b9bf222
refactor: convert VLMVerificationResult to Pydantic BaseModel; fix ty…
Jun 25, 2026
2616477
Automatically reformatting code
Jun 25, 2026
da816d0
Generate VLM verification model from spec; move ask_vlm_verify to exp…
Jun 29, 2026
a38eb92
Automatically reformatting code
Jun 29, 2026
729ae3c
regen: run full make generate with Java — add VlmVerificationsApi + m…
Jun 29, 2026
89a91e3
fix: restore cli.py to main (only drop the ask_vlm command)
Jun 29, 2026
3c90b08
simplify: build vlm-verifications URL like create_note (no version st…
Jun 29, 2026
9d23339
fix: import VlmVerification from model, not groundlight (mypy)
Jun 29, 2026
c031c1b
fix: exclude ask_vlm_verify from CLI auto-registration
Jun 29, 2026
e574ae0
Add ask_vlm_verify as an experimental CLI command
Jun 29, 2026
1c4a496
docs: note CLI supports only a single image for now
Jun 29, 2026
532e3cb
Potential fix for pull request finding
srnangi Jun 29, 2026
4aa5e5e
Use generated VlmVerificationsApi instead of raw requests
Jun 29, 2026
7154463
Automatically reformatting code
Jun 29, 2026
09ed731
Use one-indexed media filenames (image_1.jpg, image_2.jpg, ...)
Jun 29, 2026
01ff309
docs(spec): trim vlm-verifications endpoint description
Jul 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
trim ask_vlm tests to meaningful coverage only
Drop tests that only verify kwarg passthroughs or mock server-side
behavior (timeout forwarding, corrupted-image 400, dual-image loop,
model_id omission). Keep the five that catch real issues or verify
non-obvious invariants: result parsing, image encoding, form-field
vs URL security property, >8 guard, and the URL path bug.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
  • Loading branch information
buildci and claude committed Jun 24, 2026
commit 7216313d8b6cd0b5a9e57e6e9a1c032496ce263b
70 changes: 12 additions & 58 deletions test/unit/test_ask_vlm.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def _mock_response(verdict="YES", confidence=0.92, reasoning="Flames visible.",


def test_returns_vlm_verification_result(gl: Groundlight):
"""ask_vlm returns a typed VLMVerificationResult with all expected fields populated."""
"""Result fields are correctly unpacked from the server response JSON."""
with mock.patch("groundlight.client.requests") as mock_requests:
mock_requests.post.return_value = _mock_response()
result = gl.ask_vlm(media=np.zeros((100, 100, 3), dtype=np.uint8), query="Is there a fire?")
Expand All @@ -45,8 +45,8 @@ def test_returns_vlm_verification_result(gl: Groundlight):
assert result.total_cost_usd == pytest.approx(0.0015)


def test_single_numpy_image_encoded_as_jpeg(gl: Groundlight):
"""A numpy array is encoded to JPEG and sent as a single multipart 'media' part."""
def test_numpy_image_encoded_as_jpeg_multipart(gl: Groundlight):
"""A numpy array is converted to JPEG and sent as a multipart 'media' part."""
with mock.patch("groundlight.client.requests") as mock_requests:
mock_requests.post.return_value = _mock_response()
gl.ask_vlm(media=np.zeros((480, 640, 3), dtype=np.uint8), query="Is there a fire?")
Expand All @@ -60,33 +60,9 @@ def test_single_numpy_image_encoded_as_jpeg(gl: Groundlight):
assert len(data) > 0


def test_dual_images_sends_two_parts(gl: Groundlight):
"""Passing a list of two images sends two 'media' multipart parts."""
with mock.patch("groundlight.client.requests") as mock_requests:
mock_requests.post.return_value = _mock_response()
gl.ask_vlm(
media=[np.zeros((480, 640, 3), dtype=np.uint8), np.zeros((120, 120, 3), dtype=np.uint8)],
query="Is there a fire?",
)

_, kwargs = mock_requests.post.call_args
assert len(kwargs["files"]) == 2


def test_url_has_correct_path(gl: Groundlight):
"""sanitize_endpoint_url strips the trailing slash, so we must insert '/' before
the path — without it the URL would be '...device-apiv1/vlm-verifications'."""
with mock.patch("groundlight.client.requests") as mock_requests:
mock_requests.post.return_value = _mock_response()
gl.ask_vlm(media=np.zeros((100, 100, 3), dtype=np.uint8), query="test")

args, _ = mock_requests.post.call_args
url = args[0]
assert "/device-api/v1/vlm-verifications" in url


def test_query_and_model_id_sent_as_form_fields(gl: Groundlight):
"""query and model_id go in the multipart body, never in the URL query string."""
def test_query_sent_as_form_field_not_url_param(gl: Groundlight):
"""query and model_id go in the multipart body — never the URL — so the prompt
doesn't leak into access logs."""
with mock.patch("groundlight.client.requests") as mock_requests:
mock_requests.post.return_value = _mock_response(model_id="nova-pro")
gl.ask_vlm(media=np.zeros((100, 100, 3), dtype=np.uint8), query="Is there a fire?", model_id="nova-pro")
Expand All @@ -97,40 +73,18 @@ def test_query_and_model_id_sent_as_form_fields(gl: Groundlight):
assert "params" not in kwargs or not kwargs.get("params")


def test_no_model_id_omits_field(gl: Groundlight):
"""Omitting model_id leaves the field out entirely so the server uses its default."""
with mock.patch("groundlight.client.requests") as mock_requests:
mock_requests.post.return_value = _mock_response()
gl.ask_vlm(media=np.zeros((100, 100, 3), dtype=np.uint8), query="test")

_, kwargs = mock_requests.post.call_args
assert "model_id" not in kwargs["data"]


def test_more_than_eight_media_raises(gl: Groundlight):
"""Supplying more than 8 media items raises ValueError before any network call."""
with pytest.raises(ValueError, match="at most 8"):
gl.ask_vlm(media=[np.zeros((100, 100, 3), dtype=np.uint8)] * 9, query="test")


def test_timeout_passed_to_requests(gl: Groundlight):
"""The timeout parameter is forwarded to requests.post."""
def test_url_has_correct_path(gl: Groundlight):
"""sanitize_endpoint_url strips the trailing slash from self.endpoint, so the path
must include a leading '/' — without it the URL becomes '...device-apiv1/...'."""
with mock.patch("groundlight.client.requests") as mock_requests:
mock_requests.post.return_value = _mock_response()
gl.ask_vlm(media=np.zeros((100, 100, 3), dtype=np.uint8), query="test", timeout=5.0)

_, kwargs = mock_requests.post.call_args
assert kwargs["timeout"] == pytest.approx(5.0)


def test_corrupted_image_bytes_raises_http_error(gl: Groundlight):
"""Corrupted bytes are not validated client-side — the server rejects them with a
400, which raise_for_status() converts to requests.HTTPError."""
error_resp = MagicMock()
error_resp.status_code = 400
error_resp.raise_for_status.side_effect = Exception("400 Bad Request")
gl.ask_vlm(media=np.zeros((100, 100, 3), dtype=np.uint8), query="test")

with mock.patch("groundlight.client.requests") as mock_requests:
mock_requests.post.return_value = error_resp
with pytest.raises(Exception, match="400"):
gl.ask_vlm(media=b"this-is-not-a-valid-image", query="test")
args, _ = mock_requests.post.call_args
assert "/device-api/v1/vlm-verifications" in args[0]
Comment thread
srnangi marked this conversation as resolved.
Outdated
Loading