Skip to content

Commit aa60b78

Browse files
Rakesh Ravi ShankarMichelle Sintov
Rakesh Ravi Shankar
authored and
Michelle Sintov
committed
VIC-4415: Adding annotations to camera feed (#149)
Added ability to use annotations on camera feed, single-shot image, and remote control view.
1 parent 09a4e23 commit aa60b78

11 files changed

Lines changed: 935 additions & 38 deletions

File tree

‎anki_vector/annotate.py‎

Lines changed: 559 additions & 0 deletions
Large diffs are not rendered by default.

‎anki_vector/camera.py‎

Lines changed: 112 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
import time
3333
import sys
3434

35-
from . import connection, util
35+
from . import annotate, connection, util
3636
from .exceptions import VectorCameraFeedException
3737
from .messaging import protocol
3838

@@ -60,6 +60,78 @@ def _convert_to_pillow_image(image_data: bytes) -> Image.Image:
6060
return image
6161

6262

63+
class CameraImage:
64+
"""A single image from the robot's camera.
65+
This wraps a raw image and provides an :meth:`annotate_image` method
66+
that can resize and add dynamic annotations to the image, such as
67+
marking up the location of objects and faces.
68+
69+
.. testcode::
70+
71+
import anki_vector
72+
73+
with anki_vector.Robot() as robot:
74+
image = robot.camera.capture_single_image()
75+
print(f"Displaying image with id {image.image_id}, received at {image.image_recv_time}")
76+
image.raw_image.show()
77+
78+
:param raw_image: The raw unprocessed image from the camera.
79+
:param image_annotator: The image annotation object.
80+
:param image_id: An image number that increments on every new image received.
81+
"""
82+
83+
def __init__(self, raw_image: Image.Image, image_annotator: annotate.ImageAnnotator, image_id: int):
84+
85+
self._raw_image = raw_image
86+
self._image_annotator = image_annotator
87+
self._image_id = image_id
88+
self._image_recv_time = time.time()
89+
90+
@property
91+
def raw_image(self) -> Image.Image:
92+
"""The raw unprocessed image from the camera."""
93+
return self._raw_image
94+
95+
@property
96+
def image_id(self) -> int:
97+
"""An image number that increments on every new image received."""
98+
return self._image_id
99+
100+
@property
101+
def image_recv_time(self) -> float:
102+
"""The time the image was received and processed by the SDK."""
103+
return self._image_recv_time
104+
105+
def annotate_image(self, scale: float = None, fit_size: tuple = None, resample_mode: int = annotate.RESAMPLE_MODE_NEAREST) -> Image.Image:
106+
"""Adds any enabled annotations to the image.
107+
Optionally resizes the image prior to annotations being applied. The
108+
aspect ratio of the resulting image always matches that of the raw image.
109+
110+
.. testcode::
111+
112+
import anki_vector
113+
114+
with anki_vector.Robot() as robot:
115+
image = robot.camera.capture_single_image()
116+
annotated_image = image.annotate_image()
117+
annotated_image.show()
118+
119+
:param scale: If set then the base image will be scaled by the
120+
supplied multiplier. Cannot be combined with fit_size
121+
:param fit_size: If set, then scale the image to fit inside
122+
the supplied (width, height) dimensions. The original aspect
123+
ratio will be preserved. Cannot be combined with scale.
124+
:param resample_mode: The resampling mode to use when scaling the
125+
image. Should be either :attr:`~anki_vector.annotate.RESAMPLE_MODE_NEAREST`
126+
(fast) or :attr:`~anki_vector.annotate.RESAMPLE_MODE_BILINEAR` (slower,
127+
but smoother).
128+
"""
129+
return self._image_annotator.annotate_image(self._raw_image,
130+
scale=scale,
131+
fit_size=fit_size,
132+
resample_mode=resample_mode)
133+
134+
63135
class CameraComponent(util.Component):
64136
"""Represents Vector's camera.
65137
@@ -75,22 +147,27 @@ class CameraComponent(util.Component):
75147
with anki_vector.Robot() as robot:
76148
robot.camera.init_camera_feed()
77149
image = robot.camera.latest_image
78-
image.show()
150+
image.raw_image.show()
79151
80152
:param robot: A reference to the owner Robot object.
81153
"""
82154

155+
#: callable: The factory function that returns an
156+
#: :class:`annotate.ImageAnnotator` class or subclass instance.
157+
annotator_factory = annotate.ImageAnnotator
158+
83159
def __init__(self, robot):
84160
super().__init__(robot)
85161

86-
self._latest_image: Image.Image = None
162+
self._image_annotator: annotate.ImageAnnotator = self.annotator_factory(self.robot.world)
163+
self._latest_image: CameraImage = None
87164
self._latest_image_id: int = None
88165
self._camera_feed_task: asyncio.Task = None
89166
self._enabled = False
90167

91168
@property
92169
@util.block_while_none()
93-
def latest_image(self) -> Image.Image:
170+
def latest_image(self) -> CameraImage:
94171
""":class:`Image.Image`: The most recently processed image received from the robot.
95172
96173
The resolution of latest_image is 640x360.
@@ -104,7 +181,7 @@ def latest_image(self) -> Image.Image:
104181
with anki_vector.Robot() as robot:
105182
robot.camera.init_camera_feed()
106183
image = robot.camera.latest_image
107-
image.show()
184+
image.raw_image.show()
108185
"""
109186
if not self._camera_feed_task:
110187
raise VectorCameraFeedException()
@@ -126,13 +203,31 @@ def latest_image_id(self) -> int:
126203
with anki_vector.Robot() as robot:
127204
robot.camera.init_camera_feed()
128205
image = robot.camera.latest_image
129-
image.show()
206+
image.raw_image.show()
130207
print(f"latest_image_id: {robot.camera.latest_image_id}")
131208
"""
132209
if not self._camera_feed_task:
133210
raise VectorCameraFeedException()
134211
return self._latest_image_id
135212

213+
@property
214+
def image_annotator(self) -> annotate.ImageAnnotator:
215+
"""The image annotator used to add annotations to the raw camera images.
216+
217+
.. testcode::
218+
219+
import time
220+
import anki_vector
221+
222+
with anki_vector.Robot(show_viewer=True) as robot:
223+
# Annotations (enabled by default) are displayed on the camera feed
224+
time.sleep(5)
225+
# Disable all annotations
226+
robot.camera.image_annotator.annotation_enabled = False
227+
time.sleep(5)
228+
"""
229+
return self._image_annotator
230+
136231
def init_camera_feed(self) -> None:
137232
"""Begin camera feed task.
138233
@@ -143,7 +238,7 @@ def init_camera_feed(self) -> None:
143238
with anki_vector.Robot() as robot:
144239
robot.camera.init_camera_feed()
145240
image = robot.camera.latest_image
146-
image.show()
241+
image.raw_image.show()
147242
"""
148243
if not self._camera_feed_task or self._camera_feed_task.done():
149244
self._enabled = True
@@ -203,11 +298,14 @@ def image_streaming_enabled(self) -> bool:
203298
def _unpack_image(self, msg: protocol.CameraFeedResponse) -> None:
204299
"""Processes raw data from the robot into a more useful image structure."""
205300
image = _convert_to_pillow_image(msg.data)
206-
self.robot.viewer.enqueue_frame(image)
207301

208-
self._latest_image = image
302+
self._latest_image = CameraImage(image, self._image_annotator, msg.image_id)
209303
self._latest_image_id = msg.image_id
210304

305+
if self._image_annotator.annotation_enabled:
306+
image = self._image_annotator.annotate_image(image)
307+
self.robot.viewer.enqueue_frame(image)
308+
211309
async def _request_and_handle_images(self) -> None:
212310
"""Queries and listens for camera feed events from the robot.
213311
Received events are parsed by a helper function."""
@@ -224,7 +322,7 @@ async def _request_and_handle_images(self) -> None:
224322
self.logger.debug('Camera feed task was cancelled. This is expected during disconnection.')
225323

226324
@connection.on_connection_thread()
227-
async def capture_single_image(self) -> Image.Image:
325+
async def capture_single_image(self) -> CameraImage:
228326
"""Request to capture a single image from the robot's camera.
229327
230328
This call requests the robot to capture an image and returns the
@@ -239,12 +337,14 @@ async def capture_single_image(self) -> Image.Image:
239337
240338
with anki_vector.Robot() as robot:
241339
image = robot.camera.capture_single_image()
242-
image.show()
340+
image.raw_image.show()
243341
"""
244342
if self._enabled:
245343
return self._latest_image
246344
req = protocol.CaptureSingleImageRequest()
247345
res = await self.grpc_interface.CaptureSingleImage(req)
248346
if res and res.data:
249-
return _convert_to_pillow_image(res.data)
347+
image = _convert_to_pillow_image(res.data)
348+
return CameraImage(image, self._image_annotator, res.image_id)
349+
250350
self.logger.error('Failed to capture a single image')

‎anki_vector/objects.py‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -861,7 +861,7 @@ def descriptive_name(self) -> str:
861861
cube = robot.world.connected_light_cube
862862
print(f"{cube.descriptive_name}")
863863
"""
864-
return "{0} id={1} factory_id={2} is_connected={3}".format(self.__class__.__name__, self._object_id, self._factory_id, self._is_connected)
864+
return f"{self.__class__.__name__}\nid={self._object_id}\nfactory_id={self._factory_id}\nis_connected={self._is_connected}"
865865

866866
@property
867867
def object_id(self) -> int:
@@ -1045,6 +1045,23 @@ def object_id(self, value: str):
10451045
self.logger.debug("Setting object_id for %s to %s", self.__class__, value)
10461046
self._object_id = value
10471047

1048+
@property
1049+
def descriptive_name(self) -> str:
1050+
"""A descriptive name for this ObservableObject instance.
1051+
1052+
Note: Sub-classes should override this to add any other relevant info
1053+
for that object type.
1054+
1055+
.. testcode::
1056+
1057+
import anki_vector
1058+
1059+
with anki_vector.Robot() as robot:
1060+
if robot.world.charger.is_visible:
1061+
print(f"{obot.world.charger.descriptive_name}")
1062+
"""
1063+
return f"{self.__class__.__name__} id={self._object_id}"
1064+
10481065
#### Private Methods ####
10491066

10501067
def _on_object_observed(self, _robot, _event_type, msg):

‎anki_vector/opengl/opengl.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,9 @@ def raise_opengl_or_pillow_import_error(opengl_import_exc):
6060
if isinstance(opengl_import_exc, InvalidOpenGLGlutImplementation):
6161
raise NotImplementedError('GLUT (OpenGL Utility Toolkit) is not available:\n%s'
6262
% opengl_import_exc)
63-
else:
64-
raise NotImplementedError('OpenGL is not available; '
65-
'make sure the PyOpenGL and Pillow packages are installed:\n'
66-
'Do `pip3 install --user "anki_vector[3dviewer]"` to install. Error: %s' % opengl_import_exc)
63+
raise NotImplementedError('OpenGL is not available; '
64+
'make sure the PyOpenGL and Pillow packages are installed:\n'
65+
'Do `pip3 install --user "anki_vector[3dviewer]"` to install. Error: %s' % opengl_import_exc)
6766

6867

6968
try:

‎anki_vector/robot.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ def camera(self) -> camera.CameraComponent:
232232
with anki_vector.Robot() as robot:
233233
robot.camera.init_camera_feed()
234234
image = robot.camera.latest_image
235-
image.show()
235+
image.raw_image.show()
236236
"""
237237
if self._camera is None:
238238
raise VectorNotReadyException("CameraComponent is not yet initialized")
@@ -617,7 +617,6 @@ def connect(self, timeout: int = 10) -> None:
617617
self._anim = animation.AnimationComponent(self)
618618
self._audio = audio.AudioComponent(self)
619619
self._behavior = behavior.BehaviorComponent(self)
620-
self._camera = camera.CameraComponent(self)
621620
self._faces = faces.FaceComponent(self)
622621
self._motors = motors.MotorComponent(self)
623622
self._nav_map = nav_map.NavMapComponent(self)
@@ -629,6 +628,7 @@ def connect(self, timeout: int = 10) -> None:
629628
self._viewer_3d = viewer.Viewer3DComponent(self)
630629
self._vision = vision.VisionComponent(self)
631630
self._world = world.World(self)
631+
self._camera = camera.CameraComponent(self)
632632

633633
if self.cache_animation_lists:
634634
# Load animation triggers and animations so they are ready to play when requested

‎anki_vector/util.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
from pathlib import Path
5151
import sys
5252
import time
53-
from typing import Callable
53+
from typing import Callable, Union
5454

5555
from .exceptions import VectorConfigurationException, VectorPropertyValueNotReadyException
5656
from .messaging import protocol
@@ -858,6 +858,15 @@ def height(self) -> float:
858858
"""The height of the object from when it was last visible within Vector's camera view."""
859859
return self._height
860860

861+
def scale_by(self, scale_multiplier: Union[int, float]) -> None:
862+
"""Scales the image rectangle by the multiplier provided."""
863+
if not isinstance(scale_multiplier, (int, float)):
864+
raise TypeError("Unsupported operand for * expected number")
865+
self._x_top_left *= scale_multiplier
866+
self._y_top_left *= scale_multiplier
867+
self._width *= scale_multiplier
868+
self._height *= scale_multiplier
869+
861870

862871
class Distance:
863872
"""Represents a distance.
@@ -1084,7 +1093,7 @@ def read_configuration(serial: str, logger: logging.Logger) -> dict:
10841093
sections = parser.sections()
10851094
if not sections:
10861095
raise VectorConfigurationException('Could not find the sdk configuration file. Please run `python3 -m anki_vector.configure` to set up your Vector for SDK usage.')
1087-
elif serial is None and len(sections) == 1:
1096+
if serial is None and len(sections) == 1:
10881097
serial = sections[0]
10891098
logger.warning("No serial number provided. Automatically selecting {}".format(serial))
10901099
elif serial is None:

‎anki_vector/world.py‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,24 @@ def visible_custom_objects(self) -> Iterable[objects.CustomObject]:
195195
if obj.is_visible:
196196
yield obj
197197

198+
@property
199+
def visible_objects(self) -> Iterable[objects.ObservableObject]:
200+
"""generator: yields each object that Vector can currently see.
201+
202+
.. testcode::
203+
204+
import anki_vector
205+
with anki_vector.Robot() as robot:
206+
for obj in robot.world.visible_objects:
207+
print(obj)
208+
209+
Returns:
210+
A generator yielding Charger, LightCube and CustomObject instances
211+
"""
212+
for obj in self._objects.values():
213+
if obj.is_visible:
214+
yield obj
215+
198216
@property
199217
def connected_light_cube(self) -> objects.LightCube:
200218
"""A light cube connected to Vector, if any.
@@ -843,7 +861,7 @@ def _remove_all_fixed_custom_object_instances(self):
843861

844862
def _on_face_observed(self, _robot, _event_type, msg):
845863
"""Adds a newly observed face to the world view."""
846-
if msg.face_id not in self._faces or msg.face_id not in self._objects:
864+
if msg.face_id not in self._faces:
847865
pose = util.Pose(x=msg.pose.x, y=msg.pose.y, z=msg.pose.z,
848866
q0=msg.pose.q0, q1=msg.pose.q1,
849867
q2=msg.pose.q2, q3=msg.pose.q3,
@@ -857,23 +875,24 @@ def _on_face_observed(self, _robot, _event_type, msg):
857875
msg.left_eye, msg.right_eye, msg.nose, msg.mouth, msg.timestamp)
858876
if face:
859877
self._faces[face.face_id] = face
860-
self._objects[face.face_id] = face
861878

862879
def _on_object_observed(self, _robot, _event_type, msg):
863880
"""Adds a newly observed custom object to the world view."""
864-
if msg.object_type == protocol.ObjectType.Value("BLOCK_LIGHTCUBE1"):
881+
first_custom_type = protocol.ObjectType.Value("FIRST_CUSTOM_OBJECT_TYPE")
882+
if msg.object_type == objects.LIGHT_CUBE_1_TYPE:
865883
if msg.object_id not in self._objects:
866-
if self.light_cube:
867-
self._objects[msg.object_id] = self.light_cube
884+
light_cube = self._light_cube.get(objects.LIGHT_CUBE_1_TYPE)
885+
if light_cube:
886+
light_cube.object_id = msg.object_id
887+
self._objects[msg.object_id] = light_cube
868888

869-
if msg.object_type == protocol.ObjectType.Value("CHARGER_BASIC"):
889+
elif msg.object_type == protocol.ObjectType.Value("CHARGER_BASIC"):
870890
if msg.object_id not in self._objects:
871891
charger = self._allocate_charger(msg)
872892
if charger:
873893
self._objects[msg.object_id] = charger
874894

875-
first_custom_type = protocol.ObjectType.Value("FIRST_CUSTOM_OBJECT_TYPE")
876-
if first_custom_type <= msg.object_type < (first_custom_type + protocol.CustomType.Value("CUSTOM_TYPE_COUNT")):
895+
elif first_custom_type <= msg.object_type < (first_custom_type + protocol.CustomType.Value("CUSTOM_TYPE_COUNT")):
877896
if msg.object_id not in self._objects:
878897
custom_object = self._allocate_custom_marker_object(msg)
879898
if custom_object:

‎docs/source/api.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ The API
88

99
anki_vector
1010
anki_vector.animation
11+
anki_vector.annotate
1112
anki_vector.audio
1213
anki_vector.behavior
1314
anki_vector.camera

‎docs/source/images/annotate.png‎

2.89 MB
Loading

0 commit comments

Comments
 (0)