diff --git a/newrelic/api/opentelemetry.py b/newrelic/api/opentelemetry.py index 56df7dedd..8cd009be6 100644 --- a/newrelic/api/opentelemetry.py +++ b/newrelic/api/opentelemetry.py @@ -17,6 +17,10 @@ from contextlib import contextmanager from opentelemetry import trace as otel_api_trace +from opentelemetry.baggage.propagation import W3CBaggagePropagator +from opentelemetry.propagate import set_global_textmap +from opentelemetry.propagators.composite import CompositePropagator +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator from newrelic.api.application import application_instance from newrelic.api.background_task import BackgroundTask @@ -33,21 +37,68 @@ _logger = logging.getLogger(__name__) +class NRTraceContextPropagator(TraceContextTextMapPropagator): + HEADER_KEYS = ("traceparent", "tracestate", "newrelic") + + def extract(self, carrier, context=None, getter=None): + transaction = current_transaction() + # If we are passing into New Relic, traceparent + # and/or tracestate's keys also need to be NR compatible. + + if transaction: + nr_headers = { + header_key: getter.get(carrier, header_key)[0] + for header_key in self.HEADER_KEYS + if getter.get(carrier, header_key) + } + transaction.accept_distributed_trace_headers(nr_headers) + return super().extract(carrier=carrier, context=context, getter=getter) + + def inject(self, carrier, context=None, setter=None): + transaction = current_transaction() + # Only insert headers if we have not done so already this transaction + # New Relic's Distributed Trace State will have the following states: + # 0 (00) if not set: + # Transaction has not inserted any outbound headers nor has + # it accepted any inbound headers (yet). + # 1 (01) if already accepted: + # Transaction has accepted inbound headers and is able to + # insert outbound headers to the next app if needed. + # 2 (10) if inserted but not accepted: + # Transaction has inserted outbound headers already. + # Do not insert outbound headers multiple times. This is + # a fundamental difference in OTel vs NR behavior: if + # headers are inserted by OTel multiple times, it will + # propagate the last set of data that was inserted. NR + # will not allow more than one header insertion per + # transaction. + # 3 (11) if accepted, then inserted: + # Transaction has accepted inbound headers and has inserted + # outbound headers. + + if not transaction: + return super().inject(carrier=carrier, context=context, setter=setter) + + if transaction._distributed_trace_state < 2: + nr_headers = [] + transaction.insert_distributed_trace_headers(nr_headers) + for key, value in nr_headers: + setter.set(carrier, key, value) + # Do NOT call super().inject() since we have already + # inserted the headers here. It will not cause harm, + # but it is redundant logic. + + # If distributed_trace_state == 2 or 3, do not inject headers. + + +# Context and Context Propagator Setup +otel_context_propagator = CompositePropagator(propagators=[NRTraceContextPropagator(), W3CBaggagePropagator()]) +set_global_textmap(otel_context_propagator) + # ---------------------------------------------- # Custom OTel Spans and Traces # ---------------------------------------------- -# TracerProvider: we can think of this as the agent instance. Only one can exist -# SpanProcessor: we can think of this as an application. In NR, we can have multiple applications -# though right now, we can only do SpanProcessor and SynchronousMultiSpanProcessor -# Tracer: we can think of this as the transaction. -# Span: we can think of this as the trace. -# Links functionality has now been enabled but not implemented yet. Links are relationships -# between spans, but lateral in hierarchy. In NR we only have parent-child relationships. -# We may want to preserve this information with a custom attribute. We can also add this -# as a new attribute in a trace, but it will still not be seen in the UI other than a trace -# attribute. - class Span(otel_api_trace.Span): def __init__( @@ -73,11 +124,6 @@ def __init__( self.nr_trace = None self.instrumenting_module = instrumenting_module - # Do not create a New Relic trace if parent - # is a remote span and it is not sampled - if self._remote() and not self._sampled(): - return - self.nr_parent = None current_nr_trace = current_trace() if ( @@ -100,7 +146,7 @@ def __init__( _logger.error( "OpenTelemetry span (%s) and NR trace (%s) do not match nor correspond to a remote span. Open Telemetry span will not be reported to New Relic. Please report this problem to New Relic.", self.otel_parent, - current_nr_trace, + current_nr_trace, # NR parent trace ) return @@ -140,26 +186,6 @@ def __init__( self.nr_trace.__enter__() - def _sampled(self): - # Uses NR to determine if the trace is sampled - # - # transaction.sampled can be `None`, `True`, `False`. - # If `None`, this has not been computed by NR which - # can also mean the following: - # 1. There was not a context passed in that explicitly has sampling disabled. - # This flag would be found in the traceparent or traceparent and tracespan headers. - # 2. Transaction was not created where DT headers are accepted during __init__ - # Therefore, we will treat a value of `None` as `True` for now. - # - # The primary reason for this behavior is because Otel expects to - # only be able to record information like events and attributes - # when `is_recording()` == `True` - - if self.otel_parent: - return bool(self.otel_parent.trace_flags) - else: - return bool(self.nr_transaction and (self.nr_transaction.sampled or (self.nr_transaction.sampled is None))) - def _remote(self): # Remote span denotes if propagated from a remote parent return bool(self.otel_parent and self.otel_parent.is_remote) @@ -168,14 +194,12 @@ def get_span_context(self): if not getattr(self, "nr_trace", False): return otel_api_trace.INVALID_SPAN_CONTEXT - otel_tracestate_headers = None - return otel_api_trace.SpanContext( trace_id=int(self.nr_transaction.trace_id, 16), span_id=int(self.nr_trace.guid, 16), is_remote=self._remote(), - trace_flags=otel_api_trace.TraceFlags(0x01 if self._sampled() else 0x00), - trace_state=otel_api_trace.TraceState(otel_tracestate_headers), + trace_flags=otel_api_trace.TraceFlags(0x01), + trace_state=otel_api_trace.TraceState(), ) def set_attribute(self, key, value): @@ -193,8 +217,7 @@ def _set_attributes_in_nr(self, otel_attributes=None): def add_event(self, name, attributes=None, timestamp=None): # TODO: Not implemented yet. - # We can implement this as a log event - raise NotImplementedError("TODO: We can implement this as a log event.") + raise NotImplementedError("Events are not implemented yet.") def add_link(self, context=None, attributes=None): # TODO: Not implemented yet. @@ -208,7 +231,14 @@ def update_name(self, name): self.nr_trace.name = self._name def is_recording(self): - return self._sampled() and not (getattr(self.nr_trace, None), "end_time", None) + # If the trace has an end time set then it is done recording. Otherwise, + # if it does not have an end time set and the transaction's priority + # has not been set yet or it is set to something other than 0 then it + # is also still recording. + if getattr(self.nr_trace, "end_time", None): + return False + + return getattr(self.nr_transaction, "priority", 1) > 0 def set_status(self, status, description=None): # TODO: not implemented yet @@ -277,6 +307,10 @@ def start_span( self.nr_application = application_instance() self.attributes = attributes or {} + if not self.nr_application.active: + # Force application registration if not already active + self.nr_application.activate() + if not self.nr_application.settings.otel_bridge.enabled: return otel_api_trace.INVALID_SPAN @@ -286,7 +320,12 @@ def start_span( if parent_span_context is None or not parent_span_context.is_valid: parent_span_context = None + parent_span_trace_id = None + if parent_span_context and self.nr_application.settings.distributed_tracing.enabled: + parent_span_trace_id = parent_span_context.trace_id + # If remote_parent, transaction must be created, regardless of kind type + # Make sure we transfer DT headers when we are here, if DT is enabled if parent_span_context and parent_span_context.is_remote: if kind in (otel_api_trace.SpanKind.SERVER, otel_api_trace.SpanKind.CLIENT): # This is a web request @@ -296,6 +335,7 @@ def start_span( port = self.attributes.get("net.host.port") request_method = self.attributes.get("http.method") request_path = self.attributes.get("http.route") + transaction = WebTransaction( self.nr_application, name=name, @@ -306,6 +346,7 @@ def start_span( request_path=request_path, headers=headers, ) + elif kind in (otel_api_trace.SpanKind.PRODUCER, otel_api_trace.SpanKind.INTERNAL): transaction = BackgroundTask(self.nr_application, name=name) elif kind == otel_api_trace.SpanKind.CONSUMER: @@ -315,7 +356,7 @@ def start_span( destination_name=name, application=self.nr_application, transport_type=self.instrumentation_library, - headers=headers, + headers=None, ) transaction.__enter__() @@ -348,6 +389,11 @@ def start_span( request_path=request_path, headers=headers, ) + + transaction._trace_id = ( + f"{parent_span_trace_id:x}" if parent_span_trace_id else transaction.trace_id + ) + transaction.__enter__() elif kind == otel_api_trace.SpanKind.INTERNAL: if transaction: @@ -372,7 +418,7 @@ def start_span( destination_name=name, application=self.nr_application, transport_type=self.instrumentation_library, - headers=headers, + headers=None, ) transaction.__enter__() elif kind == otel_api_trace.SpanKind.PRODUCER: diff --git a/newrelic/config.py b/newrelic/config.py index 8aaa1305c..87fec8b3f 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -4428,6 +4428,16 @@ def _process_module_builtin_defaults(): ) # Hybrid Agent Hooks + _process_module_definition( + "opentelemetry.context", "newrelic.hooks.hybridagent_opentelemetry", "instrument_context_api" + ) + + _process_module_definition( + "opentelemetry.instrumentation.propagators", + "newrelic.hooks.hybridagent_opentelemetry", + "instrument_global_propagators_api", + ) + _process_module_definition( "opentelemetry.trace", "newrelic.hooks.hybridagent_opentelemetry", "instrument_trace_api" ) diff --git a/newrelic/hooks/hybridagent_opentelemetry.py b/newrelic/hooks/hybridagent_opentelemetry.py index 41248657f..9cf800096 100644 --- a/newrelic/hooks/hybridagent_opentelemetry.py +++ b/newrelic/hooks/hybridagent_opentelemetry.py @@ -13,10 +13,11 @@ # limitations under the License. import logging +import os from newrelic.api.application import application_instance from newrelic.api.time_trace import add_custom_span_attribute, current_trace -from newrelic.api.transaction import Sentinel, current_transaction +from newrelic.api.transaction import current_transaction from newrelic.common.object_wrapper import wrap_function_wrapper from newrelic.common.signature import bind_args from newrelic.core.config import global_settings @@ -24,6 +25,56 @@ _logger = logging.getLogger(__name__) _TRACER_PROVIDER = None +# Enable OpenTelemetry Bridge to capture HTTP +# request/response headers as span attributes: +os.environ["OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST"] = ".*" +os.environ["OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE"] = ".*" + + +########################################### +# Context Instrumentation +########################################### + + +def wrap__load_runtime_context(wrapped, instance, args, kwargs): + settings = global_settings() + + if not settings.otel_bridge.enabled: + return wrapped(*args, **kwargs) + + from opentelemetry.context.contextvars_context import ContextVarsRuntimeContext + + context = ContextVarsRuntimeContext() + return context + + +def wrap_get_global_response_propagator(wrapped, instance, args, kwargs): + settings = global_settings() + + if not settings.otel_bridge.enabled: + return wrapped(*args, **kwargs) + + from opentelemetry.instrumentation.propagators import set_global_response_propagator + + from newrelic.api.opentelemetry import otel_context_propagator + + set_global_response_propagator(otel_context_propagator) + + return otel_context_propagator + + +def instrument_context_api(module): + if hasattr(module, "_load_runtime_context"): + wrap_function_wrapper(module, "_load_runtime_context", wrap__load_runtime_context) + + +def instrument_global_propagators_api(module): + # Need to disable this instrumentation if settings.otel_bridge is disabled + + if hasattr(module, "get_global_response_propagator"): + wrap_function_wrapper(module, "get_global_response_propagator", wrap_get_global_response_propagator) + + ########################################### # Trace Instrumentation ########################################### @@ -31,7 +82,6 @@ def wrap_set_tracer_provider(wrapped, instance, args, kwargs): settings = global_settings() - if not settings.otel_bridge.enabled: return wrapped(*args, **kwargs) @@ -46,18 +96,20 @@ def wrap_set_tracer_provider(wrapped, instance, args, kwargs): def wrap_get_tracer_provider(wrapped, instance, args, kwargs): + # This needs to act as a singleton, like the agent instance. + # We should initialize the agent here as well, if there is + # not an instance already. + + application = application_instance() + if not application.active: + # Force application registration if not already active + application.activate() + settings = global_settings() if not settings.otel_bridge.enabled: return wrapped(*args, **kwargs) - # This needs to act as a singleton, like the agent instance. - # We should initialize the agent here as well, if there is - # not an instance already. - application = application_instance(activate=False) - if not application or (application and not application.active): - application_instance().activate() - global _TRACER_PROVIDER if _TRACER_PROVIDER is None: @@ -71,24 +123,18 @@ def wrap_get_tracer_provider(wrapped, instance, args, kwargs): def wrap_get_current_span(wrapped, instance, args, kwargs): transaction = current_transaction() trace = current_trace() + span = wrapped(*args, **kwargs) - # If a NR trace does not exist (aside from the Sentinel - # trace), return the original function's result. - if not transaction or isinstance(trace, Sentinel): - return wrapped(*args, **kwargs) + if not transaction: + return span - # Do not allow the wrapper to continue if - # the Hybrid Agent setting is not enabled settings = transaction.settings or global_settings() - if not settings.otel_bridge.enabled: - return wrapped(*args, **kwargs) + return span # If a NR trace does exist, check to see if the current # OTel span corresponds to the current NR trace. If so, # return the original function's result. - span = wrapped(*args, **kwargs) - if span.get_span_context().span_id == int(trace.guid, 16): return span @@ -112,14 +158,20 @@ def set_attributes(self, attributes): for key, value in attributes.items(): add_custom_span_attribute(key, value) - otel_tracestate_headers = None + if transaction.settings.distributed_tracing.enabled: + if not isinstance(span, otel_api_trace.NonRecordingSpan): + # Use the Otel trace and span ids if current span + # is not a NonRecordingSpan. Otherwise, we need to + # override the current span with the transaction + # and trace guids. + transaction._trace_id = f"{span.get_span_context().trace_id:x}" span_context = otel_api_trace.SpanContext( trace_id=int(transaction.trace_id, 16), span_id=int(trace.guid, 16), is_remote=span.get_span_context().is_remote, - trace_flags=otel_api_trace.TraceFlags(span.get_span_context().trace_flags), - trace_state=otel_api_trace.TraceState(otel_tracestate_headers), + trace_flags=otel_api_trace.TraceFlags(0x01), + trace_state=otel_api_trace.TraceState(), ) return LazySpan(span_context) @@ -144,12 +196,10 @@ def wrap_start_internal_or_server_span(wrapped, instance, args, kwargs): if context_carrier: if ("HTTP_HOST" in context_carrier) or ("http_version" in context_carrier): # This is an HTTP request (WSGI, ASGI, or otherwise) - nr_environ = context_carrier.copy() - attributes["nr.http.headers"] = nr_environ + attributes["nr.http.headers"] = context_carrier else: - nr_headers = context_carrier.copy() - attributes["nr.nonhttp.headers"] = nr_headers + attributes["nr.nonhttp.headers"] = context_carrier bound_args["attributes"] = attributes diff --git a/tests/hybridagent_opentelemetry/cross_agent/TestCaseDefinitions.json b/tests/hybridagent_opentelemetry/cross_agent/TestCaseDefinitions.json new file mode 100644 index 000000000..ff91707c8 --- /dev/null +++ b/tests/hybridagent_opentelemetry/cross_agent/TestCaseDefinitions.json @@ -0,0 +1,641 @@ +[ + { + "testDescription": "Does not create segment without a transaction", + + "operations": [ + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "Bar", + "spanKind": "Internal" + }, + "assertions": [ + { + "description": "The OpenTelmetry span should not be created", + "rule": { + "operator": "NotValid", + "parameters": { + "object": "currentOTelSpan" + } + } + }, + { + "description": "There should be no transaction", + "rule": { + "operator": "NotValid", + "parameters": { + "object": "currentTransaction" + } + } + } + ] + } + ], + + "agentOutput": { + "transactions": [], + "spans": [] + } + }, + + { + "testDescription": "Creates OpenTelemetry segment in a transaction", + + "operations": [ + { + "command": "DoWorkInTransaction", + "parameters": { + "transactionName": "Foo" + }, + "childOperations": [ + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "Bar", + "spanKind": "Internal" + }, + "assertions": [ + { + "description": "OpenTelemetry API and New Relic API report the same traceId", + "rule": { + "operator": "Equals", + "parameters": { + "left": "currentOTelSpan.traceId", + "right": "currentTransaction.traceId" + } + } + }, + { + "description": "OpenTelemetry API and New Relic API report the same spanId", + "rule": { + "operator": "Equals", + "parameters": { + "left": "currentOTelSpan.spanId", + "right": "currentSegment.spanId" + } + } + } + ] + } + ] + } + ], + + "agentOutput": { + "transactions": [ + { + "name": "Foo" + } + ], + "spans": [ + { + "name": "Bar", + "category": "generic", + "parentName": "Foo" + }, + { + "name": "Foo", + "category": "generic", + "entryPoint": true + } + ] + } + }, + + { + "testDescription": "Creates New Relic span as child of OpenTelemetry span", + + "operations": [ + { + "command": "DoWorkInTransaction", + "parameters": { + "transactionName": "Foo" + }, + "childOperations": [ + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "Bar", + "spanKind": "Internal" + }, + "childOperations": [ + { + "command": "DoWorkInSegment", + "parameters": { + "segmentName": "Baz" + }, + "assertions": [ + { + "description": "OpenTelemetry API and New Relic API report the same traceId", + "rule": { + "operator": "Equals", + "parameters": { + "left": "currentOTelSpan.traceId", + "right": "currentTransaction.traceId" + } + } + }, + { + "description": "OpenTelemetry API and New Relic API report the same spanId", + "rule": { + "operator": "Equals", + "parameters": { + "left": "currentOTelSpan.spanId", + "right": "currentSegment.spanId" + } + } + } + ] + } + ] + } + ] + } + ], + + "agentOutput": { + "transactions": [ + { + "name": "Foo" + } + ], + "spans": [ + { + "name": "Baz", + "category": "generic", + "parentName": "Bar" + }, + { + "name": "Bar", + "category": "generic", + "parentName": "Foo" + }, + { + "name": "Foo", + "category": "generic", + "entryPoint": true + } + ] + } + }, + + { + "testDescription": "OpenTelemetry API can add custom attributes to spans", + + "operations": [ + { + "command": "DoWorkInTransaction", + "parameters": { + "transactionName": "Foo" + }, + "childOperations": [ + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "Bar", + "spanKind": "Internal" + }, + "childOperations": [ + { + "command": "DoWorkInSegment", + "parameters": { + "segmentName": "Baz" + }, + "childOperations": [ + { + "command": "AddOTelAttribute", + "parameters": { + "name": "spanNumber", + "value": 2 + } + } + ] + }, + { + "command": "AddOTelAttribute", + "parameters": { + "name": "spanNumber", + "value": 1 + } + } + ] + } + ] + } + ], + + "agentOutput": { + "transactions": [ + { + "name": "Foo" + } + ], + "spans": [ + { + "name": "Baz", + "attributes": { + "spanNumber": 2 + } + }, + { + "name": "Bar", + "attributes": { + "spanNumber": 1 + } + } + ] + } + }, + + { + "testDescription": "OpenTelemetry API can record errors", + + "operations": [ + { + "command": "DoWorkInTransaction", + "parameters": { + "transactionName": "Foo" + }, + "childOperations": [ + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "Bar", + "spanKind": "Internal" + }, + "childOperations": [ + { + "command": "RecordExceptionOnSpan", + "parameters": { + "errorMessage": "Test exception message" + } + } + ] + } + ] + } + ], + + "agentOutput": { + "transactions": [ + { + "name": "Foo" + } + ], + "spans": [ + { + "name": "Bar", + "attributes": { + "error.message": "Test exception message" + } + } + ] + } + }, + + { + "testDescription": "OpenTelemetry API and New Relic API can inject outbound trace context", + + "operations": [ + { + "command": "DoWorkInTransaction", + "parameters": { + "transactionName": "Foo" + }, + "childOperations": [ + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "OTelSpan1", + "spanKind": "Client" + }, + "childOperations": [ + { + "command": "SimulateExternalCall", + "parameters": { + "url": "url1" + }, + "childOperations": [ + { + "command": "OTelInjectHeaders", + "assertions": [ + { + "description": "Correct traceId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.traceId", + "right": "currentTransaction.traceId" + } + } + }, + { + "description": "Correct spanId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.spanId", + "right": "currentSegment.spanId" + } + } + }, + { + "description": "Correct sampled flag was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.sampled", + "right": "currentTransaction.sampled" + } + } + } + ] + } + ] + } + ] + }, + { + "command": "DoWorkInSegment", + "parameters": { + "segmentName": "segment1" + }, + "childOperations": [ + { + "command": "SimulateExternalCall", + "parameters": { + "url": "url2" + }, + "childOperations": [ + { + "command": "OTelInjectHeaders", + "assertions": [ + { + "description": "Correct traceId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.traceId", + "right": "currentTransaction.traceId" + } + } + }, + { + "description": "Correct spanId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.spanId", + "right": "currentSegment.spanId" + } + } + }, + { + "description": "Correct sampled flag was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.sampled", + "right": "currentTransaction.sampled" + } + } + } + ] + } + ] + } + ] + }, + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "OTelSpan2", + "spanKind": "Client" + }, + "childOperations": [ + { + "command": "SimulateExternalCall", + "parameters": { + "url": "url3" + }, + "childOperations": [ + { + "command": "NRInjectHeaders", + "assertions": [ + { + "description": "Correct traceId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.traceId", + "right": "currentTransaction.traceId" + } + } + }, + { + "description": "Correct spanId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.spanId", + "right": "currentSegment.spanId" + } + } + }, + { + "description": "Correct sampled flag was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.sampled", + "right": "currentTransaction.sampled" + } + } + } + ] + } + ] + } + ] + }, + { + "command": "DoWorkInSegment", + "parameters": { + "segmentName": "segment2" + }, + "childOperations": [ + { + "command": "SimulateExternalCall", + "parameters": { + "url": "url4" + }, + "childOperations": [ + { + "command": "NRInjectHeaders", + "assertions": [ + { + "description": "Correct traceId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.traceId", + "right": "currentTransaction.traceId" + } + } + }, + { + "description": "Correct spanId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.spanId", + "right": "currentSegment.spanId" + } + } + }, + { + "description": "Correct sampled flag was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.sampled", + "right": "currentTransaction.sampled" + } + } + } + ] + } + ] + } + ] + } + ] + } + ], + + "agentOutput": { + "transactions": [ + { + "name": "Foo" + } + ], + "spans": [ + { + "name": "OTelSpan1", + "parentName": "Foo" + }, + { + "name": "segment1", + "parentName": "Foo" + }, + { + "name": "OTelSpan2", + "parentName": "Foo" + }, + { + "name": "segment2", + "parentName": "Foo" + } + ] + } + }, + + { + "testDescription": "Starting transaction tests", + + "operations": [ + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "Foo", + "spanKind": "Server" + } + }, + { + "command": "DoWorkInSpanWithRemoteParent", + "parameters": { + "spanName": "Bar", + "spanKind": "Server" + } + }, + { + "command": "DoWorkInTransaction", + "parameters": { + "transactionName": "Baz" + }, + "childOperations": [ + { + "command": "DoWorkInSpanWithRemoteParent", + "parameters": { + "spanName": "EdgeCase", + "spanKind": "Server" + } + } + ] + } + ], + + "agentOutput": { + "transactions": [ + { + "name": "Foo" + }, + { + "name": "Bar" + }, + { + "name": "Baz" + } + ], + "spans": [ + { + "name": "EdgeCase", + "parentName": "Baz" + }, + { + "name": "Baz", + "entryPoint": true + } + ] + } + }, + + { + "testDescription": "Inbound distributed tracing tests", + + "operations": [ + { + "command": "DoWorkInSpanWithInboundContext", + "parameters": { + "spanName": "Foo", + "spanKind": "Server", + "traceIdInHeader": "da8bc8cc6d062849b0efcf3c169afb5a", + "spanIdInHeader": "7d3efb1b173fecfa", + "sampledFlagInHeader": "0" + }, + "assertions": [ + { + "description": "Current span has expected traceId", + "rule": { + "operator": "Matches", + "parameters": { + "object": "currentOTelSpan.traceId", + "value": "da8bc8cc6d062849b0efcf3c169afb5a" + } + } + } + ] + } + ], + + "agentOutput": { + "transactions": [ + { + "name": "Foo" + } + ], + "spans": [] + } + } + +] diff --git a/tests/hybridagent_opentelemetry/test_traces_attributes.py b/tests/hybridagent_opentelemetry/test_attributes.py similarity index 64% rename from tests/hybridagent_opentelemetry/test_traces_attributes.py rename to tests/hybridagent_opentelemetry/test_attributes.py index b9f370593..77ebee68f 100644 --- a/tests/hybridagent_opentelemetry/test_traces_attributes.py +++ b/tests/hybridagent_opentelemetry/test_attributes.py @@ -25,8 +25,8 @@ def test_trace_with_span_attributes(tracer): @validate_span_events( count=1, exact_intrinsics={ - "name": "Function/test_traces_attributes:test_trace_with_span_attributes.._test", - "transaction.name": "OtherTransaction/Function/test_traces_attributes:test_trace_with_span_attributes.._test", + "name": "Function/test_attributes:test_trace_with_span_attributes.._test", + "transaction.name": "OtherTransaction/Function/test_attributes:test_trace_with_span_attributes.._test", "sampled": True, }, ) @@ -49,50 +49,59 @@ def test_trace_with_otel_to_newrelic(tracer): This test adds custom attributes to the transaction and trace. * `add_custom_attribute` adds custom attributes to the transaction. * `add_custom_span_attribute` adds custom attributes to the trace. - NOTE: a transaction's custom attributes are added to the root - span's user attributes. + NOTE: + 1. Distinction between trace and span attributes is given for + whether they are added to the transaction or the span. A + transaction's custom attributes are added to the root span's + user attributes. + 2. Notation for attributes: + - NR trace attributes: "NR_trace_attribute_" + - NR span attributes: "NR_span_attribute_" + - OTel span attributes: "otel_span_attribute_" + Where is either `FT` or `BG`, for FunctionTrace + or BackgroundTask, respectively. """ @function_trace() def newrelic_function_trace(): - add_custom_attribute("NR_trace_attribute_from_function", "NR trace attribute") - add_custom_span_attribute("NR_span_attribute_from_function", "NR span attribute") + add_custom_attribute("NR_trace_attribute_FT", "NR trace attribute from FT") + add_custom_span_attribute("NR_span_attribute_FT", "NR span attribute from FT") otel_span = trace.get_current_span() - otel_span.set_attribute("otel_span_attribute_from_function", "OTel span attribute from FT") + otel_span.set_attribute("otel_span_attribute_FT", "OTel span attribute from FT") @validate_span_events( count=1, exact_intrinsics={ - "name": "Function/test_traces_attributes:test_trace_with_otel_to_newrelic..newrelic_background_task", - "transaction.name": "OtherTransaction/Function/test_traces_attributes:test_trace_with_otel_to_newrelic..newrelic_background_task", + "name": "Function/test_attributes:test_trace_with_otel_to_newrelic..newrelic_background_task", + "transaction.name": "OtherTransaction/Function/test_attributes:test_trace_with_otel_to_newrelic..newrelic_background_task", "sampled": True, }, - exact_users={"NR_trace_attribute_from_function": "NR trace attribute"}, + exact_users={"NR_trace_attribute_FT": "NR trace attribute from FT"}, ) @validate_span_events( count=1, exact_intrinsics={"name": "Function/foo", "sampled": True}, expected_intrinsics={"priority": None, "traceId": None, "guid": None}, exact_users={ - "nr_trace_attribute": "NR span attribute from BG", + "NR_span_attribute_BG": "NR span attribute from BG", "otel_span_attribute_BG": "OTel span attribute from BG", }, ) @validate_span_events( count=1, exact_intrinsics={ - "name": "Function/test_traces_attributes:test_trace_with_otel_to_newrelic..newrelic_function_trace", + "name": "Function/test_attributes:test_trace_with_otel_to_newrelic..newrelic_function_trace", "sampled": True, }, exact_users={ - "NR_span_attribute_from_function": "NR span attribute", - "otel_span_attribute_from_function": "OTel span attribute from FT", + "NR_span_attribute_FT": "NR span attribute from FT", + "otel_span_attribute_FT": "OTel span attribute from FT", }, ) @background_task() def newrelic_background_task(): with tracer.start_as_current_span("foo") as otel_span: - add_custom_span_attribute("nr_trace_attribute", "NR span attribute from BG") + add_custom_span_attribute("NR_span_attribute_BG", "NR span attribute from BG") otel_span.set_attribute("otel_span_attribute_BG", "OTel span attribute from BG") newrelic_function_trace() diff --git a/tests/hybridagent_opentelemetry/test_context_propagation.py b/tests/hybridagent_opentelemetry/test_context_propagation.py new file mode 100644 index 000000000..48ae73dc3 --- /dev/null +++ b/tests/hybridagent_opentelemetry/test_context_propagation.py @@ -0,0 +1,125 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from opentelemetry import propagate as otel_api_propagate +from opentelemetry import trace as otel_api_trace +from testing_support.fixtures import override_application_settings +from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes + +from newrelic.api.background_task import background_task +from newrelic.api.transaction import accept_distributed_trace_headers, current_transaction + +PROPAGATOR = otel_api_propagate.get_global_textmap() + +_override_settings = {"trusted_account_key": "1", "distributed_tracing.enabled": True, "span_events.enabled": True} + + +# @dt_enabled +@pytest.mark.parametrize("telemetry", ["newrelic", "otel"]) +@pytest.mark.parametrize("propagation", [accept_distributed_trace_headers, PROPAGATOR.extract]) +def test_distributed_trace_header_compatibility_full_granularity(telemetry, propagation): + """ + Args: + telemetry (str): either "newrelic", or "otel" + Denotes which propagation function was used to + insert/inject the distributed trace headers. + "newrelic" => `insert_distributed_trace_headers` + "otel" => `PROPAGATOR.inject` from OTel API + propagation (func): The propagation function to use. + Either `accept_distributed_trace_headers` + or `PROPAGATOR.extract`. + """ + + @override_application_settings(_override_settings) + @validate_transaction_event_attributes( + required_params={ + "agent": [], + "user": [], + "intrinsic": [ + "priority" # Ensure that priority is set, even if only traceparent is passed. + ], + } + ) + @background_task() + def _test(): + transaction = current_transaction() + + headers = { + "traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01", + "newrelic": '{"v":[0,1],"d":{"ty":"Mobile","ac":"123","ap":"51424","id":"5f474d64b9cc9b2a","tr":"6e2fea0b173fdad0","pr":0.1234,"sa":True,"ti":1482959525577,"tx":"27856f70d3d314b7"}}', # This header should be ignored. + } + if telemetry == "newrelic": + headers["tracestate"] = "1@nr=0-0-1-2827902-0af7651916cd43dd-00f067aa0ba902b7-1-1.23456-1518469636035" + # "otel" does not generate tracestate headers + + propagation(headers) + current_span = otel_api_trace.get_current_span() + assert transaction.parent_span == "00f067aa0ba902b7" + assert transaction.trace_id == "0af7651916cd43dd8448eb211c80319c" + assert current_span.get_span_context().trace_id == int("0af7651916cd43dd8448eb211c80319c", 16) + + _test() + + +@pytest.mark.parametrize("telemetry", ["newrelic", "otel"]) +@pytest.mark.parametrize("propagation", [accept_distributed_trace_headers, PROPAGATOR.extract]) +def test_distributed_trace_header_compatibility_partial_granularity(telemetry, propagation): + """ + Args: + telemetry (str): either "newrelic" or "otel" + Denotes which propagation function was used to + insert/inject the distributed trace headers. + "newrelic" => `insert_distributed_trace_headers` + "otel" => `PROPAGATOR.inject` from Otel API + propagation (func): The propagation function to use. + Either `accept_distributed_trace_headers` + or `PROPAGATOR.extract`. + """ + test_settings = _override_settings.copy() + test_settings.update( + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + } + ) + + @override_application_settings(test_settings) + @validate_transaction_event_attributes( + required_params={ + "agent": [], + "user": [], + "intrinsic": [ + "priority" # Ensure that priority is set, even if only traceparent is passed. + ], + } + ) + @background_task() + def _test(): + transaction = current_transaction() + headers = { + "traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01", + "newrelic": '{"v":[0,1],"d":{"ty":"Mobile","ac":"123","ap":"51424","id":"5f474d64b9cc9b2a","tr":"6e2fea0b173fdad0","pr":0.1234,"sa":True,"ti":1482959525577,"tx":"27856f70d3d314b7"}}', # This header should be ignored. + } + if telemetry == "newrelic": + headers["tracestate"] = "1@nr=0-0-1-2827902-0af7651916cd43dd-00f067aa0ba902b7-1-1.23456-1518469636035" + # "otel" does not generate tracestate headers + + propagation(headers) + current_span = otel_api_trace.get_current_span() + assert transaction.parent_span == "00f067aa0ba902b7" + assert transaction.trace_id == "0af7651916cd43dd8448eb211c80319c" + assert current_span.get_span_context().trace_id == int("0af7651916cd43dd8448eb211c80319c", 16) + + _test() diff --git a/tests/hybridagent_opentelemetry/test_hybrid_cross_agent.py b/tests/hybridagent_opentelemetry/test_hybrid_cross_agent.py index 9a31d2bf6..f2de9a748 100644 --- a/tests/hybridagent_opentelemetry/test_hybrid_cross_agent.py +++ b/tests/hybridagent_opentelemetry/test_hybrid_cross_agent.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from opentelemetry import propagate as otel_api_propagate from opentelemetry import trace as otel_api_trace +from testing_support.fixtures import dt_enabled, override_application_settings from testing_support.validators.validate_error_event_attributes import validate_error_event_attributes from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_count import validate_transaction_count @@ -20,10 +22,13 @@ from newrelic.api.application import application_instance from newrelic.api.background_task import BackgroundTask +from newrelic.api.external_trace import ExternalTrace from newrelic.api.function_trace import FunctionTrace from newrelic.api.time_trace import current_trace from newrelic.api.transaction import current_transaction +PROPAGATOR = otel_api_propagate.get_global_textmap() + # Does not create segment without a transaction @validate_transaction_count(0) @@ -37,6 +42,7 @@ def test_does_not_create_segment_without_a_transaction(tracer): # Creates OpenTelemetry segment in a transaction +@dt_enabled @validate_transaction_metrics(name="Foo", background_task=True) @validate_span_events( exact_intrinsics={"name": "Function/Bar", "category": "generic"}, expected_intrinsics=("parentId",) @@ -49,7 +55,7 @@ def test_creates_opentelemetry_segment_in_a_transaction(tracer): with tracer.start_as_current_span(name="Bar", kind=otel_api_trace.SpanKind.INTERNAL): # OpenTelemetry API and New Relic API report the same traceId assert otel_api_trace.get_current_span().get_span_context().trace_id == int( - current_transaction()._trace_id, 16 + current_transaction().trace_id, 16 ) # OpenTelemetry API and New Relic API report the same spanId @@ -57,6 +63,7 @@ def test_creates_opentelemetry_segment_in_a_transaction(tracer): # Creates New Relic span as child of OpenTelemetry span +@dt_enabled @validate_transaction_metrics(name="Foo", background_task=True) @validate_span_events( exact_intrinsics={"name": "Function/Baz", "category": "generic"}, expected_intrinsics=("parentId",) @@ -64,6 +71,7 @@ def test_creates_opentelemetry_segment_in_a_transaction(tracer): @validate_span_events( exact_intrinsics={"name": "Function/Bar", "category": "generic"}, expected_intrinsics=("parentId",) ) +@validate_span_events(exact_intrinsics={"name": "Function/Foo", "category": "generic", "nr.entryPoint": True}) @validate_span_events(exact_intrinsics={"name": "Function/Foo", "category": "generic"}) def test_creates_new_relic_span_as_child_of_open_telemetry_span(tracer): application = application_instance(activate=False) @@ -81,6 +89,7 @@ def test_creates_new_relic_span_as_child_of_open_telemetry_span(tracer): # OpenTelemetry API can add custom attributes to spans +@dt_enabled @validate_transaction_metrics(name="Foo", background_task=True) @validate_span_events(exact_intrinsics={"name": "Function/Baz"}, exact_users={"spanNumber": 2}) @validate_span_events(exact_intrinsics={"name": "Function/Bar"}, exact_users={"spanNumber": 1}) @@ -96,6 +105,7 @@ def test_opentelemetry_api_can_add_custom_attributes_to_spans(tracer): # OpenTelemetry API can record errors +@dt_enabled @validate_transaction_metrics(name="Foo", background_task=True) @validate_error_event_attributes( exact_attrs={"agent": {}, "intrinsic": {"error.message": "Test exception message"}, "user": {}} @@ -110,3 +120,145 @@ def test_opentelemetry_api_can_record_errors(tracer): raise Exception("Test exception message") except Exception as e: otel_api_trace.get_current_span().record_exception(e) + + +# OpenTelemetry API and New Relic API can inject outbound trace context +@dt_enabled +@validate_transaction_metrics(name="Foo", background_task=True) +@validate_span_events(exact_intrinsics={"name": "External/url1/OtelSpan1/GET"}, expected_intrinsics=("parentId",)) +@validate_span_events(exact_intrinsics={"name": "External/url2/segment1/GET"}, expected_intrinsics=("parentId",)) +@validate_span_events(exact_intrinsics={"name": "External/url3/OtelSpan2/GET"}, expected_intrinsics=("parentId",)) +@validate_span_events(exact_intrinsics={"name": "External/url4/segment2/GET"}, expected_intrinsics=("parentId",)) +def test_opentelemetry_api_and_new_relic_api_can_inject_outbound_trace_context(tracer): + application = application_instance(activate=False) + + with BackgroundTask(application, name="Foo"): + transaction = current_transaction() + with tracer.start_as_current_span( + name="OtelSpan1", + kind=otel_api_trace.SpanKind.CLIENT, + attributes={"http.url": "http://url1", "http.method": "GET"}, + ): + headers = {} + PROPAGATOR.inject(carrier=headers) + _, trace_id, span_id, sampled = headers["traceparent"].split("-") + + # Correct traceId was injected + assert transaction.trace_id == trace_id + + # Correct spanId was injected + assert current_trace().guid == span_id + + # Correct sampled flag was injected + assert transaction.sampled == (sampled == "01") + + # Reset the distributed trace state for the purposes of this test + transaction._distributed_trace_state = 0 + + with ExternalTrace(library="segment1", url="http://url2", method="GET"): + headers = {} + PROPAGATOR.inject(carrier=headers) + _, trace_id, span_id, sampled = headers["traceparent"].split("-") + + # Correct traceId was injected + assert current_transaction().trace_id == trace_id + + # Correct spanId was injected + assert current_trace().guid == span_id + + # Correct sampled flag was injected + assert current_transaction().sampled == (sampled == "01") + + # Reset the distributed trace state for the purposes of this test + transaction._distributed_trace_state = 0 + + with tracer.start_as_current_span( + name="OtelSpan2", + kind=otel_api_trace.SpanKind.CLIENT, + attributes={"http.url": "http://url3", "http.method": "GET"}, + ): + headers = [] + transaction.insert_distributed_trace_headers(headers) + _, trace_id, span_id, sampled = headers[0][1].split("-") + + # Correct traceId was injected + assert transaction.trace_id == trace_id + + # Correct spanId was injected + assert current_trace().guid == span_id + + # Correct sampled flag was injected + assert transaction.sampled == (sampled == "01") + + # Reset the distributed trace state for the purposes of this test + transaction._distributed_trace_state = 0 + + with ExternalTrace(library="segment2", url="http://url4", method="GET"): + headers = [] + transaction.insert_distributed_trace_headers(headers) + _, trace_id, span_id, sampled = headers[0][1].split("-") + + # Correct traceId was injected + assert current_transaction().trace_id == trace_id + + # Correct spanId was injected + assert current_trace().guid == span_id + + # Correct sampled flag was injected + assert current_transaction().sampled == (sampled == "01") + + +# Starting transaction tests +@dt_enabled +@validate_transaction_metrics(name="Foo", index=-3) +@validate_transaction_metrics(name="Bar", index=-2) +@validate_transaction_metrics(name="Baz", background_task=True, index=-1) +@validate_span_events(exact_intrinsics={"name": "Function/EdgeCase"}, expected_intrinsics=("parentId",)) +@validate_span_events(exact_intrinsics={"name": "Function/Baz", "nr.entryPoint": True}) +def test_starting_transaction_tests(tracer): + application = application_instance(activate=False) + + with tracer.start_as_current_span(name="Foo", kind=otel_api_trace.SpanKind.SERVER): + pass + + # Create remote span context and remote context + remote_span_context = otel_api_trace.SpanContext( + trace_id=0x1234567890ABCDEF1234567890ABCDEF, + span_id=0x1234567890ABCDEF, + is_remote=True, + trace_flags=otel_api_trace.TraceFlags.SAMPLED, + trace_state=otel_api_trace.TraceState(), + ) + remote_context = otel_api_trace.set_span_in_context(otel_api_trace.NonRecordingSpan(remote_span_context)) + + with tracer.start_as_current_span(name="Bar", kind=otel_api_trace.SpanKind.SERVER, context=remote_context): + pass + + with BackgroundTask(application, name="Baz"): + with tracer.start_as_current_span(name="EdgeCase", kind=otel_api_trace.SpanKind.SERVER, context=remote_context): + pass + + +# Inbound distributed tracing tests +@dt_enabled +@validate_transaction_metrics(name="Foo") +@validate_span_events(count=0) +@override_application_settings({"trusted_account_key": "1", "account_id": "1"}) +def test_inbound_distributed_tracing_tests(tracer): + """ + This test intends to check for a scenario where an external call + span is made outside the context of an existing transaction. + By flagging that span as not sampled in OTel, it means that the OTel + api will reflect that our agent is also ignoring that span. In + this case, it will create the server transaction, but no spans. + """ + with tracer.start_as_current_span(name="Foo", kind=otel_api_trace.SpanKind.SERVER): + carrier = { + "traceparent": "00-da8bc8cc6d062849b0efcf3c169afb5a-7d3efb1b173fecfa-00", + "tracestate": "1@nr=0-0-1-12345678-7d3efb1b173fecfa-da8bc8cc6d062849-0-0.23456-1011121314151", + } + PROPAGATOR.extract(carrier=carrier) + + current_span = otel_api_trace.get_current_span() + + assert current_span.get_span_context().trace_id == 0xDA8BC8CC6D062849B0EFCF3C169AFB5A diff --git a/tests/hybridagent_opentelemetry/test_settings.py b/tests/hybridagent_opentelemetry/test_settings.py index 1ccd109ab..7e0af1857 100644 --- a/tests/hybridagent_opentelemetry/test_settings.py +++ b/tests/hybridagent_opentelemetry/test_settings.py @@ -13,8 +13,9 @@ # limitations under the License. import pytest -from testing_support.fixtures import override_application_settings +from testing_support.fixtures import dt_enabled, override_application_settings from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from newrelic.api.background_task import background_task from newrelic.api.time_trace import current_trace @@ -22,9 +23,10 @@ @pytest.mark.parametrize("enabled", [True, False]) -def test_distributed_tracing_enabled(tracer, enabled): +def test_opentelemetry_bridge_enabled(tracer, enabled): @override_application_settings({"otel_bridge.enabled": enabled}) - @validate_span_events(count=1, exact_intrinsics={"name": "Function/Foo"}) + @dt_enabled + @validate_transaction_metrics(name="Foo", background_task=True) @validate_span_events(count=1 if enabled else 0, exact_intrinsics={"name": "Function/Bar"}) @validate_span_events(count=1 if enabled else 0, exact_intrinsics={"name": "Function/Baz"}) @background_task(name="Foo")