Skip to content

Commit

Permalink
add limits
Browse files Browse the repository at this point in the history
  • Loading branch information
Andrew Xue committed Jun 2, 2020
1 parent e6e2d4b commit 9aa997f
Show file tree
Hide file tree
Showing 3 changed files with 166 additions and 26 deletions.
5 changes: 1 addition & 4 deletions docs-requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,4 @@ sqlalchemy>=1.0
thrift>=0.10.0
wrapt>=1.0.0,<2.0.0
psutil~=5.7.0
google-cloud-core >=1.3.0
google-api-core >=1.17.0
google-cloud-trace >=0.23.0
grpcio >=1.28.1
google-cloud-trace >=0.23.0
72 changes: 60 additions & 12 deletions ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,17 @@
import opentelemetry.trace as trace_api
from opentelemetry.sdk.trace import Event
from opentelemetry.sdk.trace.export import Span, SpanExporter, SpanExportResult
from opentelemetry.sdk.util import BoundedDict
from opentelemetry.util import types

logger = logging.getLogger(__name__)

MAX_NUM_LINKS = 128
MAX_NUM_EVENTS = 32
MAX_EVENT_ATTRS = 4
MAX_LINK_ATTRS = 32
MAX_SPAN_ATTRS = 32


class CloudTraceSpanExporter(SpanExporter):
"""Cloud Trace span exporter for OpenTelemetry.
Expand Down Expand Up @@ -129,7 +136,11 @@ def _translate_to_cloud_trace(
start_time = _get_time_from_ns(span.start_time)
end_time = _get_time_from_ns(span.end_time)

attributes = _extract_attributes(span.attributes)
if len(span.attributes) > MAX_SPAN_ATTRS:
logger.warning(
"Span has more then %s attributes, some will be truncated",
MAX_SPAN_ATTRS,
)

cloud_trace_spans.append(
{
Expand All @@ -141,7 +152,9 @@ def _translate_to_cloud_trace(
"start_time": start_time,
"end_time": end_time,
"parent_span_id": parent_id,
"attributes": attributes,
"attributes": _extract_attributes(
span.attributes, MAX_SPAN_ATTRS
),
"links": _extract_links(span.links),
"status": _extract_status(span.status),
"time_events": _extract_events(span.events),
Expand Down Expand Up @@ -185,7 +198,9 @@ def _get_truncatable_str_object(str_to_convert: str, max_length: int):

def _truncate_str(str_to_check: str, limit: int) -> Tuple[str, int]:
"""Check the length of a string. If exceeds limit, then truncate it."""
return str_to_check[:limit], max(0, len(str_to_check) - limit)
encoded = str_to_check.encode("utf-8")
truncated_str = encoded[:limit].decode("utf-8", errors="ignore")
return truncated_str, len(encoded) - len(truncated_str.encode("utf-8"))


def _extract_status(status: trace_api.Status) -> Status:
Expand All @@ -205,30 +220,56 @@ def _extract_links(links: Sequence[trace_api.Link]) -> ProtoSpan.Links:
if not links:
return None
extracted_links = []
dropped_links = 0
if len(links) > MAX_NUM_LINKS:
logger.warning(
"Exporting more then %s links, some will be truncated",
MAX_NUM_LINKS,
)
dropped_links = len(links) - MAX_NUM_LINKS
links = links[:MAX_NUM_LINKS]
for link in links:
if len(link.attributes) > MAX_LINK_ATTRS:
logger.warning(
"Link has more then %s attributes, some will be truncated",
MAX_LINK_ATTRS,
)
trace_id = _get_hexadecimal_trace_id(link.context.trace_id)
span_id = _get_hexadecimal_span_id(link.context.span_id)
extracted_links.append(
{
"trace_id": trace_id,
"span_id": span_id,
"type": "TYPE_UNSPECIFIED",
"attributes": _extract_attributes(link.attributes),
"attributes": _extract_attributes(
link.attributes, MAX_LINK_ATTRS
),
}
)
return ProtoSpan.Links(link=extracted_links, dropped_links_count=0)
return ProtoSpan.Links(
link=extracted_links, dropped_links_count=dropped_links
)


def _extract_events(events: Sequence[Event]) -> ProtoSpan.TimeEvents:
"""Convert span.events to dict."""
if not events:
return None
logs = []
dropped_annontations = 0
if len(events) > MAX_NUM_EVENTS:
logger.warning(
"Exporting more then %s annotations, some will be truncated",
MAX_NUM_EVENTS,
)
dropped_annontations = len(events) - MAX_NUM_EVENTS
events = events[:MAX_NUM_EVENTS]
for event in events:
if len(event.attributes) > 4:
if len(event.attributes) > MAX_EVENT_ATTRS:
logger.warning(
"Event %s has more then 4 attributes, some will be truncated",
"Event %s has more then %s attributes, some will be truncated",
event.name,
MAX_EVENT_ATTRS,
)
logs.append(
{
Expand All @@ -237,28 +278,35 @@ def _extract_events(events: Sequence[Event]) -> ProtoSpan.TimeEvents:
"description": _get_truncatable_str_object(
event.name, 256
),
"attributes": _extract_attributes(event.attributes),
"attributes": _extract_attributes(
event.attributes, MAX_EVENT_ATTRS
),
},
}
)
return ProtoSpan.TimeEvents(
time_event=logs,
dropped_annotations_count=0,
dropped_annotations_count=dropped_annontations,
dropped_message_events_count=0,
)


def _extract_attributes(attrs: types.Attributes) -> ProtoSpan.Attributes:
def _extract_attributes(
attrs: types.Attributes, num_attrs_limit: int
) -> ProtoSpan.Attributes:
"""Convert span.attributes to dict."""
attributes_dict = {}
attributes_dict = BoundedDict(num_attrs_limit)

for key, value in attrs.items():
key = _truncate_str(key, 128)[0]
value = _format_attribute_value(value)

if value is not None:
attributes_dict[key] = value
return ProtoSpan.Attributes(attribute_map=attributes_dict)
return ProtoSpan.Attributes(
attribute_map=attributes_dict,
dropped_attributes_count=len(attrs) - len(attributes_dict),
)


def _format_attribute_value(value: types.AttributeValue) -> AttributeValue:
Expand Down
115 changes: 105 additions & 10 deletions ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@
from google.rpc.status_pb2 import Status

from opentelemetry.exporter.cloud_trace import (
MAX_EVENT_ATTRS,
MAX_LINK_ATTRS,
MAX_NUM_EVENTS,
MAX_NUM_LINKS,
CloudTraceSpanExporter,
_extract_attributes,
_extract_events,
Expand All @@ -47,7 +51,6 @@ def setUp(self):
"bool_key": False,
"double_key": 1.421,
"int_key": 123,
"int_key2": 1234,
}
self.extracted_attributes_variety_pack = ProtoSpan.Attributes(
attribute_map={
Expand All @@ -63,7 +66,6 @@ def setUp(self):
)
),
"int_key": AttributeValue(int_value=123),
"int_key2": AttributeValue(int_value=1234),
}
)

Expand Down Expand Up @@ -141,18 +143,24 @@ def test_extract_status(self):

def test_extract_attributes(self):
self.assertEqual(
_extract_attributes({}), ProtoSpan.Attributes(attribute_map={})
_extract_attributes({}, 4), ProtoSpan.Attributes(attribute_map={})
)
self.assertEqual(
_extract_attributes(self.attributes_variety_pack),
_extract_attributes(self.attributes_variety_pack, 4),
self.extracted_attributes_variety_pack,
)
# Test ignoring attributes with illegal value type
self.assertEqual(
_extract_attributes({"illegal_attribute_value": dict()}),
ProtoSpan.Attributes(attribute_map={}),
_extract_attributes({"illegal_attribute_value": dict()}, 4),
ProtoSpan.Attributes(attribute_map={}, dropped_attributes_count=1),
)

too_many_attrs = {}
for attr_key in range(5):
too_many_attrs[str(attr_key)] = 0
proto_attrs = _extract_attributes(too_many_attrs, 4)
self.assertEqual(proto_attrs.dropped_attributes_count, 1)

def test_extract_events(self):
self.assertIsNone(_extract_events([]))
time_in_ns1 = 1589919268850900051
Expand Down Expand Up @@ -189,7 +197,7 @@ def test_extract_events(self):
value="event2", truncated_byte_count=0
),
"attributes": ProtoSpan.Attributes(
attribute_map={}
attribute_map={}, dropped_attributes_count=1
),
},
},
Expand Down Expand Up @@ -249,20 +257,27 @@ def test_extract_links(self):
"attributes": {
"attribute_map": {
"int_attr_value": AttributeValue(int_value=123)
}
},
"dropped_attributes_count": 1,
},
},
]
),
)

def test_truncate_string(self):
# pylint:disable=too-many-locals
def test_truncate(self):
"""Cloud Trace API imposes limits on the length of many things,
e.g. strings, number of events, number of attributes. We truncate
these things before sending it to the API as an optimization.
"""
str_300 = "a" * 300
str_256 = "a" * 256
str_128 = "a" * 128
self.assertEqual(_truncate_str("aaaa", 1), ("a", 3))
self.assertEqual(_truncate_str("aaaa", 5), ("aaaa", 0))
self.assertEqual(_truncate_str("aaaa", 4), ("aaaa", 0))
self.assertEqual(_truncate_str("中文翻译", 4), ("中", 9))

self.assertEqual(
_format_attribute_value(str_300),
Expand All @@ -272,8 +287,9 @@ def test_truncate_string(self):
)
),
)

self.assertEqual(
_extract_attributes({str_300: str_300}),
_extract_attributes({str_300: str_300}, 4),
ProtoSpan.Attributes(
attribute_map={
str_128: AttributeValue(
Expand Down Expand Up @@ -304,3 +320,82 @@ def test_truncate_string(self):
]
),
)

trace_id = "6e0c63257de34c92bf9efcd03927272e"
span_id = "95bb5edabd45950f"
link = Link(
context=SpanContext(
trace_id=int(trace_id, 16),
span_id=int(span_id, 16),
is_remote=False,
),
attributes={},
)
too_many_links = [link] * (MAX_NUM_LINKS + 1)
self.assertEqual(
_extract_links(too_many_links),
ProtoSpan.Links(
link=[
{
"trace_id": trace_id,
"span_id": span_id,
"type": "TYPE_UNSPECIFIED",
"attributes": {},
}
]
* MAX_NUM_LINKS,
dropped_links_count=len(too_many_links) - MAX_NUM_LINKS,
),
)

link_attrs = {}
for attr_key in range(MAX_LINK_ATTRS + 1):
link_attrs[str(attr_key)] = 0
attr_link = Link(
context=SpanContext(
trace_id=int(trace_id, 16),
span_id=int(span_id, 16),
is_remote=False,
),
attributes=link_attrs,
)

proto_link = _extract_links([attr_link])
self.assertEqual(
len(proto_link.link[0].attributes.attribute_map), MAX_LINK_ATTRS
)

too_many_events = [event1] * (MAX_NUM_EVENTS + 1)
self.assertEqual(
_extract_events(too_many_events),
ProtoSpan.TimeEvents(
time_event=[
{
"time": time_in_ms_and_ns1,
"annotation": {
"description": TruncatableString(
value=str_256, truncated_byte_count=300 - 256
),
"attributes": {},
},
},
]
* MAX_NUM_EVENTS,
dropped_annotations_count=len(too_many_events)
- MAX_NUM_EVENTS,
),
)

time_in_ns1 = 1589919268850900051
event_attrs = {}
for attr_key in range(MAX_EVENT_ATTRS + 1):
event_attrs[str(attr_key)] = 0
proto_events = _extract_events(
[Event(name="a", attributes=event_attrs, timestamp=time_in_ns1)]
)
self.assertEqual(
len(
proto_events.time_event[0].annotation.attributes.attribute_map
),
MAX_EVENT_ATTRS,
)

0 comments on commit 9aa997f

Please sign in to comment.