From 27bf6cae97a01329353a0fb5c6a719b417c2d767 Mon Sep 17 00:00:00 2001 From: jean-philippe bempel Date: Fri, 5 Jun 2026 08:54:04 +0200 Subject: [PATCH 01/10] Migrate dd-trace-core groovy files to java part 12 we migrate 20 tests: - TraceMapperV04PayloadTest - TraceMapperV05PayloadTest - TraceMapperV1PayloadTest - DDEvpProxyApiTest - DDIntakeApiTest - DDIntakeTraceInterceptorTest - DDIntakeTrackTypeResolverTest - DDAgentApiTest - DDAgentWriterCombinedTest - DDAgentWriterTest - DDIntakeWriterCombinedTest - DDIntakeWriterTest - MultiWriterTest - PayloadDispatcherImplTest - PrioritizationTest - SerializationTest - SpanSamplingWorkerTest - TraceMapperTest - TraceProcessingWorkerTest - WriterFactoryTest # Conflicts: # utils/junit-utils/src/main/java/datadog/trace/junit/utils/tabletest/SamplingMechanismConverter.java # Conflicts: # dd-trace-core/src/test/groovy/datadog/trace/common/writer/TraceMapperTest.groovy # dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.groovy # Conflicts: # dd-trace-core/src/test/groovy/datadog/trace/common/writer/PayloadDispatcherImplTest.groovy # Conflicts: # dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentApiTest.groovy # dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy # dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java # dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV05PayloadTest.java # dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java --- .../common/writer/TraceProcessingWorker.java | 6 + .../trace/common/writer/DDAgentApiTest.groovy | 538 ----- .../writer/DDAgentWriterCombinedTest.groovy | 753 ------- .../common/writer/DDAgentWriterTest.groovy | 222 -- .../writer/DDIntakeWriterCombinedTest.groovy | 769 ------- .../common/writer/DDIntakeWriterTest.groovy | 215 -- .../common/writer/MultiWriterTest.groovy | 69 - .../writer/PayloadDispatcherImplTest.groovy | 179 -- .../common/writer/PrioritizationTest.groovy | 258 --- .../common/writer/SerializationTest.groovy | 49 - .../writer/SpanSamplingWorkerTest.groovy | 397 ---- .../common/writer/TraceMapperTest.groovy | 101 - .../writer/TraceProcessingWorkerTest.groovy | 430 ---- .../common/writer/WriterFactoryTest.groovy | 242 -- .../writer/ddintake/DDEvpProxyApiTest.groovy | 339 --- .../writer/ddintake/DDIntakeApiTest.groovy | 360 --- .../DDIntakeTraceInterceptorTest.groovy | 89 - .../DDIntakeTrackTypeResolverTest.groovy | 24 - .../trace/common/writer/DDAgentApiTest.java | 742 +++++++ .../writer/DDAgentWriterCombinedTest.java | 837 +++++++ .../common/writer/DDAgentWriterTest.java | 226 ++ .../writer/DDIntakeWriterCombinedTest.java | 825 +++++++ .../common/writer/DDIntakeWriterTest.java | 222 ++ .../trace/common/writer/MultiWriterTest.java | 79 + .../writer/PayloadDispatcherImplTest.java | 228 ++ .../common/writer/PrioritizationTest.java | 307 +++ .../common/writer/SerializationTest.java | 65 + .../common/writer/SpanSamplingWorkerTest.java | 373 ++++ .../trace/common/writer/TraceMapperTest.java | 124 ++ .../writer/TraceProcessingWorkerTest.java | 645 ++++++ .../common/writer/WriterFactoryTest.java | 295 +++ .../ddagent/TraceMapperV04PayloadTest.java | 257 ++- .../ddagent/TraceMapperV05PayloadTest.java | 251 ++- .../ddagent/TraceMapperV1PayloadTest.java | 1948 ++++++++++++----- .../writer/ddintake/DDEvpProxyApiTest.java | 457 ++++ .../writer/ddintake/DDIntakeApiTest.java | 463 ++++ .../DDIntakeTraceInterceptorTest.java | 101 + .../DDIntakeTrackTypeResolverTest.java | 30 + .../tabletest/TableTestTypeConverters.java | 26 + 39 files changed, 7745 insertions(+), 5796 deletions(-) delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentApiTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentWriterCombinedTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentWriterTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDIntakeWriterCombinedTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDIntakeWriterTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/MultiWriterTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/PayloadDispatcherImplTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/PrioritizationTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/SerializationTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/SpanSamplingWorkerTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/TraceMapperTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/TraceProcessingWorkerTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/WriterFactoryTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDIntakeApiTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDIntakeTraceInterceptorTest.groovy delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDIntakeTrackTypeResolverTest.groovy create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterCombinedTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterCombinedTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/MultiWriterTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/PayloadDispatcherImplTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/PrioritizationTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/SerializationTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/SpanSamplingWorkerTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/TraceMapperTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/WriterFactoryTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeApiTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeTraceInterceptorTest.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeTrackTypeResolverTest.java diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/TraceProcessingWorker.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/TraceProcessingWorker.java index 39430eac71c..9b70b09bc9e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/TraceProcessingWorker.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/TraceProcessingWorker.java @@ -9,6 +9,7 @@ import datadog.common.queue.Queues; import datadog.communication.ddagent.DroppingPolicy; import datadog.trace.api.Config; +import datadog.trace.api.internal.VisibleForTesting; import datadog.trace.bootstrap.instrumentation.api.SpanPostProcessor; import datadog.trace.common.sampling.SingleSpanSampler; import datadog.trace.common.writer.ddagent.FlushEvent; @@ -122,6 +123,11 @@ public long getRemainingCapacity() { return primaryQueue.remainingCapacity(); } + @VisibleForTesting + MessagePassingBlockingQueue getPrimaryQueue() { + return primaryQueue; + } + private static MessagePassingBlockingQueue createQueue(int capacity) { return Queues.mpscBlockingConsumerArrayQueue(capacity); } diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentApiTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentApiTest.groovy deleted file mode 100644 index 0306de5ae39..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentApiTest.groovy +++ /dev/null @@ -1,538 +0,0 @@ -package datadog.trace.common.writer - -import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer -import static datadog.trace.api.ProtocolVersion.V0_4 -import static datadog.trace.api.ProtocolVersion.V0_5 -import static datadog.trace.api.ProtocolVersion.V1_0 - -import com.fasterxml.jackson.core.type.TypeReference -import com.fasterxml.jackson.databind.ObjectMapper -import datadog.communication.ddagent.DDAgentFeaturesDiscovery -import datadog.communication.http.OkHttpUtils -import datadog.communication.serialization.ByteBufferConsumer -import datadog.communication.serialization.FlushingBuffer -import datadog.communication.serialization.msgpack.MsgPackWriter -import datadog.metrics.api.Monitoring -import datadog.metrics.api.statsd.StatsDClient -import datadog.metrics.impl.MonitoringImpl -import datadog.trace.api.Config -import datadog.trace.api.ProcessTags -import datadog.trace.api.ProtocolVersion -import datadog.trace.api.config.OtlpConfig -import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags -import datadog.trace.common.sampling.RateByServiceTraceSampler -import datadog.trace.common.writer.ddagent.DDAgentApi -import datadog.trace.common.writer.ddagent.TraceMapper -import datadog.trace.common.writer.ddagent.TraceMapperV0_4 -import datadog.trace.common.writer.ddagent.TraceMapperV0_5 -import datadog.trace.common.writer.ddagent.TraceMapperV1 -import datadog.trace.core.DDSpan -import datadog.trace.core.DDSpanContext -import datadog.trace.core.propagation.PropagationTags -import datadog.trace.core.test.DDCoreSpecification -import java.nio.ByteBuffer -import java.util.concurrent.RejectedExecutionException -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicLong -import java.util.concurrent.atomic.AtomicReference -import okhttp3.HttpUrl -import okhttp3.OkHttpClient -import org.msgpack.jackson.dataformat.MessagePackFactory -import spock.lang.Shared -import spock.lang.Timeout - -@Timeout(20) -class DDAgentApiTest extends DDCoreSpecification { - - @Shared - Monitoring monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS) - static mapper = new ObjectMapper(new MessagePackFactory()) - - static newAgent(String latestVersion) { - httpServer { - handlers { - put(latestVersion) { - if (request.contentType != "application/msgpack") { - response.status(400).send("wrong type: $request.contentType") - } else if (request.contentLength <= 0) { - response.status(400).send("no content") - } else { - response.status(200).send() - } - } - } - } - } - - - def "sending an empty list of traces returns no errors"() { - setup: - def agent = newAgent(agentVersion) - def client = createAgentApi(agent.address.toString(), protocolVersion)[1] - def payload = prepareTraces(agentVersion, []) - - expect: - def response = client.sendSerializedTraces(payload) - response.success() - response.status().present - response.status().asInt == 200 - agent.getLastRequest().path == "/" + agentVersion - - cleanup: - agent.close() - - where: - agentVersion | protocolVersion - "v0.3/traces" | V0_4 - "v0.4/traces" | V0_4 - "v0.5/traces" | V0_5 - "v1.0/traces" | V1_0 - } - - def "response body propagated in case of non-200 response"() { - setup: - def agent = httpServer { - handlers { - put("v0.4/traces") { - response.status(400).send("Test error") - } - } - } - def client = createAgentApi(agent.address.toString())[1] - Payload payload = prepareTraces("v0.4/traces", []) - - expect: - def clientResponse = client.sendSerializedTraces(payload) - !clientResponse.success() - clientResponse.status().present - clientResponse.status().asInt == 400 - clientResponse.response == "Test error" - - cleanup: - agent.close() - } - - def "non-200 response"() { - setup: - def agent = httpServer { - handlers { - put("v0.4/traces") { - response.status(404).send() - } - - put("v0.3/traces") { - response.status(404).send() - } - } - } - def client = createAgentApi(agent.address.toString())[1] - Payload payload = prepareTraces("v0.3/traces", []) - expect: - def clientResponse = client.sendSerializedTraces(payload) - !clientResponse.success() - clientResponse.status().present - clientResponse.status().asInt == 404 - agent.getLastRequest().path == "/v0.3/traces" - - cleanup: - agent.close() - } - - def "content is sent as MSGPACK"() { - setup: - def agent = httpServer { - handlers { - put(agentVersion) { - response.send() - } - } - } - def client = createAgentApi(agent.address.toString())[1] - def payload = prepareTraces(agentVersion, traces) - - expect: - client.sendSerializedTraces(payload).success() - agent.lastRequest.contentType == "application/msgpack" - agent.lastRequest.headers.get("Datadog-Client-Computed-Top-Level") == "true" - agent.lastRequest.headers.get("Datadog-Meta-Lang") == "java" - agent.lastRequest.headers.get("Datadog-Meta-Lang-Version") == System.getProperty("java.version", "unknown") - agent.lastRequest.headers.get("Datadog-Meta-Tracer-Version") == "Stubbed-Test-Version" - agent.lastRequest.headers.get("X-Datadog-Trace-Count") == "${traces.size()}" - agent.lastRequest.headers.get("Datadog-Client-Dropped-P0-Traces") == "${payload.droppedTraces()}" - agent.lastRequest.headers.get("Datadog-Client-Dropped-P0-Spans") == "${payload.droppedSpans()}" - convertList(agentVersion, agent.lastRequest.body) == expectedRequestBody - - cleanup: - agent.close() - - // Populate thread info dynamically as it is different when run via gradle vs idea. - where: - // spotless:off - traces | expectedRequestBody - [] | [] - // service propagation enabled - [[buildSpan(1L, "service.name", "my-service", PropagationTags.factory().fromHeaderValue(PropagationTags.HeaderType.DATADOG, "_dd.p.usr=123"))]] | [[new TreeMap<>([ - "duration" : 10, - "error" : 0, - "meta" : ["thread.name": Thread.currentThread().getName(), "_dd.p.usr": "123", "_dd.p.dm": "-1", "_dd.p.ksr": "1", "_dd.svc_src" : "m"] + - (Config.get().isExperimentalPropagateProcessTagsEnabled() ? ["_dd.tags.process" : ProcessTags.getTagsForSerialization().toString()] : []), - "metrics" : [ - (DDSpanContext.PRIORITY_SAMPLING_KEY) : 1, - (InstrumentationTags.DD_TOP_LEVEL as String) : 1, - (RateByServiceTraceSampler.SAMPLING_AGENT_RATE): 1.0, - "thread.id" : Thread.currentThread().id - ], - "name" : "fakeOperation", - "parent_id": 0, - "resource" : "fakeResource", - "service" : "my-service", - "span_id" : 1, - "start" : 1000, - "trace_id" : 1, - "type" : "fakeType" - ])]] - // service propagation disabled - [[buildSpan(100L, "resource.name", "my-resource", PropagationTags.factory().fromHeaderValue(PropagationTags.HeaderType.DATADOG, "_dd.p.usr=123"))]] | [[new TreeMap<>([ - "duration" : 10, - "error" : 0, - "meta" : ["thread.name": Thread.currentThread().getName(), "_dd.p.usr": "123", "_dd.p.dm": "-1", "_dd.p.ksr": "1"] + - (Config.get().isExperimentalPropagateProcessTagsEnabled() ? ["_dd.tags.process" : ProcessTags.getTagsForSerialization().toString()] : []), - "metrics" : [ - (DDSpanContext.PRIORITY_SAMPLING_KEY) : 1, - (InstrumentationTags.DD_TOP_LEVEL as String) : 1, - (RateByServiceTraceSampler.SAMPLING_AGENT_RATE): 1.0, - "thread.id" : Thread.currentThread().id - ], - "name" : "fakeOperation", - "parent_id": 0, - "resource" : "my-resource", - "service" : "fakeService", - "span_id" : 1, - "start" : 100000, - "trace_id" : 1, - "type" : "fakeType" - ])]] - // spotless:on - - ignore = traces.each { - it.each { - it.finish() - it.@durationNano = 10 - } - } - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.4/traces"] - } - - def "Api ResponseListeners see 200 responses"() { - setup: - def agentResponse = new AtomicReference(null) - RemoteResponseListener responseListener = { String endpoint, Map responseJson -> - agentResponse.set(responseJson) - } - def agent = httpServer { - handlers { - put(agentVersion) { - def status = request.contentLength > 0 ? 200 : 500 - response.status(status).send('{"hello":{}}') - } - } - } - def client = createAgentApi(agent.address.toString())[1] - client.addResponseListener(responseListener) - def payload = prepareTraces(agentVersion, [[], [], []]) - payload.withDroppedTraces(1) - payload.withDroppedTraces(3) - - when: - client.sendSerializedTraces(payload) - then: - agentResponse.get() == ["hello": [:]] - agent.lastRequest.headers.get("Datadog-Meta-Lang") == "java" - agent.lastRequest.headers.get("Datadog-Meta-Lang-Version") == System.getProperty("java.version", "unknown") - agent.lastRequest.headers.get("Datadog-Meta-Tracer-Version") == "Stubbed-Test-Version" - agent.lastRequest.headers.get("X-Datadog-Trace-Count") == "3" // false data shows the value provided via traceCounter. - agent.lastRequest.headers.get("Datadog-Client-Dropped-P0-Traces") == "${payload.droppedTraces()}" - agent.lastRequest.headers.get("Datadog-Client-Dropped-P0-Spans") == "${payload.droppedSpans()}" - - cleanup: - agent.close() - - where: - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.5/traces"] - } - - def "Api Downgrades to v3 if v0.4 not available"() { - setup: - def v3Agent = httpServer { - handlers { - put("v0.3/traces") { - def status = request.contentLength > 0 ? 200 : 500 - response.status(status).send() - } - } - } - def client = createAgentApi(v3Agent.address.toString())[1] - def payload = prepareTraces("v0.4/traces", []) - expect: - client.sendSerializedTraces(payload).success() - v3Agent.getLastRequest().path == "/v0.3/traces" - - cleanup: - v3Agent.close() - } - - def "Api Downgrades to v3 if timeout exceeded (#delayTrace, #badPort)"() { - // This test is unfortunately only exercising the read timeout, not the connect timeout. - setup: - def agent = httpServer { - handlers { - put("v0.3/traces") { - def status = request.contentLength > 0 ? 200 : 500 - response.status(status).send() - } - put("v0.4/traces") { - Thread.sleep(delayTrace) - def status = request.contentLength > 0 ? 200 : 500 - response.status(status).send() - } - } - } - def port = badPort ? 999 : agent.address.port - def client = createAgentApi("http://" + agent.address.host + ":" + port)[1] - def payload = prepareTraces("v0.4/traces", []) - def result = client.sendSerializedTraces(payload) - - expect: - result.success() == !badPort // Expect success of port is ok - if (!badPort) { - assert agent.getLastRequest().path == "/$endpointVersion/traces" - } - - cleanup: - agent.close() - - where: - endpointVersion | delayTrace | badPort - "v0.4" | 0 | false - "v0.3" | 0 | true - "v0.4" | 500 | false - "v0.3" | 30000 | false - } - - def "verify content length"() { - setup: - def receivedContentLength = new AtomicLong() - def agent = httpServer { - handlers { - put(agentVersion) { - receivedContentLength.set(request.contentLength) - response.status(200).send() - } - } - } - def client = createAgentApi(agent.address.toString())[1] - def payload = prepareTraces(agentVersion, traces) - when: - def success = client.sendSerializedTraces(payload).success() - then: - success - receivedContentLength.get() == expectedLength - - cleanup: - agent.close() - - // all the tested traces are empty (why?) and it just so happens that - // arrays and maps take the same amount of space in messagepack, so - // all the sizes match, except in v0.5 where there is 1 byte for a - // 2 element array header and 1 byte for an empty dictionary - where: - // spotless:off - agentVersion | expectedLength | traces - "v0.4/traces" | 1 | [] - "v0.4/traces" | 3 | [[], []] - "v0.4/traces" | 16 | (1..15).collect { [] } - "v0.4/traces" | 19 | (1..16).collect { [] } - "v0.4/traces" | 65538 | (1..((1 << 16) - 1)).collect { [] } - "v0.4/traces" | 65541 | (1..(1 << 16)).collect { [] } - "v0.5/traces" | 1 + 1 + 1 | [] - "v0.5/traces" | 3 + 1 + 1 | [[], []] - "v0.5/traces" | 16 + 1 + 1 | (1..15).collect { [] } - "v0.5/traces" | 19 + 1 + 1 | (1..16).collect { [] } - "v0.5/traces" | 65538 + 1 + 1 | (1..((1 << 16) - 1)).collect { [] } - "v0.5/traces" | 65541 + 1 + 1 | (1..(1 << 16)).collect { [] } - // spotless:on - } - - def "Embedded HTTP client rejects async requests"() { - setup: - def agent = newAgent("v0.5/traces") - def (discovery, client) = createAgentApi(agent.address.toString()) - discovery.discover() - def httpExecutorService = client.httpClient.dispatcher().executorService() - when: - httpExecutorService.execute({}) - then: - thrown RejectedExecutionException - and: - httpExecutorService.isShutdown() - cleanup: - agent.close() - } - - void 'test metaStruct support on the encoded spans'() { - setup: - def agentVersion = 'v0.4/traces' - def meta1 = 'Hello World!' - def meta2 = [Hello: ' World!'] - def agent = httpServer { - handlers { - put(agentVersion) { - response.send() - } - } - } - def client = createAgentApi(agent.address.toString())[1] - def span = buildSpan(1L, "fakeType", [:]) - .setMetaStruct('meta_1', meta1) - .setMetaStruct('meta_2', meta2) - def payload = prepareTraces(agentVersion, [[span]]) - - expect: - client.sendSerializedTraces(payload).success() - def body = convertList(agentVersion, agent.lastRequest.body)[0][0] - def metaStruct = body['meta_struct'] as Map - assert metaStruct.size() == 2 - assert mapper.readValue(metaStruct['meta_1'], String) == meta1 - assert mapper.readValue(metaStruct['meta_2'], Map) == meta2 - - cleanup: - agent.close() - } - - static List>> convertList(String agentVersion, byte[] bytes) { - if (agentVersion.equals("v0.5/traces")) { - return convertListV5(bytes) - } - def returnVal = mapper.readValue(bytes, new TypeReference>>>() {}) - returnVal.each { - it.each { - it["meta"].remove("runtime-id") - it["meta"].remove("language") - } - } - - return returnVal - } - - static List>> convertListV5(byte[] bytes) { - List>> traces = mapper.readValue(bytes, new TypeReference>>>() {}) - List>> maps = new ArrayList<>(traces.size()) - for (List> trace : traces) { - List> mapTrace = new ArrayList<>() - for (List span : trace) { - TreeMap map = new TreeMap<>() - if (!span.isEmpty()) { - map.put("service", span.get(0)) - map.put("name", span.get(1)) - map.put("resource", span.get(2)) - map.put("trace_id", span.get(3)) - map.put("span_id", span.get(4)) - map.put("parent_id", span.get(5)) - map.put("start", span.get(6)) - map.put("duration", span.get(7)) - map.put("error", span.get(8)) - map.put("meta", span.get(9)) - map.put("metrics", span.get(10)) - map.put("type", span.get(11)) - - map.get("meta").remove("runtime-id") - map.get("meta").remove("language") - } - mapTrace.add(map) - } - maps.add(mapTrace) - } - return maps - } - - def "Datadog-Client-Computed-Stats header set when either stats pipeline is enabled (otlpSpanMetrics=#otlpSpanMetrics, nativeMetrics=#nativeMetrics)"() { - setup: - injectSysConfig(OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED, "$otlpSpanMetrics") - - def agent = httpServer { - handlers { - put("v0.4/traces") { - response.status(200).send() - } - } - } - def agentUrl = HttpUrl.get(agent.address.toString()) - - // Mock feature discovery so the native-stats pipeline signal can be controlled independently of - // what the (embedded) agent advertises. supportsMetrics() reflects agent-side client stats support. - def discovery = Mock(DDAgentFeaturesDiscovery) - discovery.getTraceEndpoint() >> "v0.4/traces" - discovery.supportsMetrics() >> nativeMetrics - discovery.state() >> null - - def client = OkHttpUtils.buildHttpClient(agentUrl, 1000) - // nativeMetricsEnabled is the constructor flag WriterFactory sets from Config.isTracerMetricsEnabled(). - def api = new DDAgentApi(client, agentUrl, discovery, monitoring, nativeMetrics) - def payload = prepareTraces("v0.4/traces", []) - - when: - // Named clientResponse (not response) to avoid shadowing the httpServer handler closure's `response`. - def clientResponse = api.sendSerializedTraces(payload) - - then: - clientResponse.success() - (agent.lastRequest.headers.get("Datadog-Client-Computed-Stats") == "true") == expectedComputesStats - - cleanup: - agent.close() - - where: - otlpSpanMetrics | nativeMetrics | expectedComputesStats - true | false | true // gap case: OTLP span metrics on, native stats off - false | false | false // neither pipeline computes stats - false | true | true // native stats on (regression guard) - true | true | true // both on - } - - Payload prepareTraces(String agentVersion, List> traces) { - Traces traceCapture = new Traces() - def packer = new MsgPackWriter(new FlushingBuffer(1 << 20, traceCapture)) - - TraceMapper traceMapper = [ - "v1.0/traces": new TraceMapperV1(), - "v0.5/traces": new TraceMapperV0_5(), - ].get(agentVersion, new TraceMapperV0_4()) - - for (trace in traces) { - packer.format(trace, traceMapper) - } - packer.flush() - - return traceMapper.newPayload() - .withBody(traceCapture.traceCount, - traces.isEmpty() ? ByteBuffer.allocate(0) : traceCapture.buffer) - } - - static class Traces implements ByteBufferConsumer { - int traceCount - ByteBuffer buffer - - @Override - void accept(int messageCount, ByteBuffer buffer) { - this.buffer = buffer - this.traceCount = messageCount - } - } - - def createAgentApi(String url, ProtocolVersion protocolVersion = V0_5) { - HttpUrl agentUrl = HttpUrl.get(url) - OkHttpClient client = OkHttpUtils.buildHttpClient(agentUrl, 1000) - DDAgentFeaturesDiscovery discovery = new DDAgentFeaturesDiscovery(client, monitoring, agentUrl, protocolVersion, true, false) - return [discovery, new DDAgentApi(client, agentUrl, discovery, monitoring, false)] - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentWriterCombinedTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentWriterCombinedTest.groovy deleted file mode 100644 index 1659ab44da3..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentWriterCombinedTest.groovy +++ /dev/null @@ -1,753 +0,0 @@ -package datadog.trace.common.writer - -import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer -import static datadog.trace.api.ProtocolVersion.V0_5 -import static datadog.trace.api.config.GeneralConfig.EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED -import static datadog.trace.common.writer.DDAgentWriter.BUFFER_SIZE -import static datadog.trace.common.writer.ddagent.Prioritization.ENSURE_TRACE - -import datadog.communication.ddagent.DDAgentFeaturesDiscovery -import datadog.communication.http.OkHttpUtils -import datadog.communication.serialization.ByteBufferConsumer -import datadog.communication.serialization.FlushingBuffer -import datadog.communication.serialization.Mapper -import datadog.communication.serialization.msgpack.MsgPackWriter -import datadog.metrics.api.statsd.StatsDClient -import datadog.metrics.impl.MonitoringImpl -import datadog.trace.api.DDSpanId -import datadog.trace.api.DDTraceId -import datadog.trace.api.ProcessTags -import datadog.trace.api.datastreams.NoopPathwayContext -import datadog.trace.api.sampling.PrioritySampling -import datadog.trace.common.writer.ddagent.DDAgentApi -import datadog.trace.common.writer.ddagent.TraceMapperV0_4 -import datadog.trace.common.writer.ddagent.TraceMapperV0_5 -import datadog.trace.core.CoreTracer -import datadog.trace.core.DDSpan -import datadog.trace.core.DDSpanContext -import datadog.trace.core.PendingTrace -import datadog.trace.core.monitor.HealthMetrics -import datadog.trace.core.monitor.TracerHealthMetrics -import datadog.trace.core.propagation.PropagationTags -import datadog.trace.core.test.DDCoreSpecification -import datadog.trace.test.util.Flaky -import java.nio.ByteBuffer -import java.util.concurrent.CountDownLatch -import java.util.concurrent.Phaser -import java.util.concurrent.Semaphore -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicInteger -import okhttp3.HttpUrl -import spock.lang.Timeout -import spock.util.concurrent.PollingConditions - -@Timeout(10) -class DDAgentWriterCombinedTest extends DDCoreSpecification { - - def conditions = new PollingConditions(timeout: 5, initialDelay: 0, factor: 1.25) - def monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS) - def phaser = new Phaser() - - // Only used to create spans - def dummyTracer = tracerBuilder().writer(new ListWriter()).build() - - def apiWithVersion(String version) { - return Mock(DDAgentApi) - } - - def setup() { - // Register for two threads. - phaser.register() - phaser.register() - } - - def cleanup() { - dummyTracer?.close() - } - - def "no interactions because of initial flush"() { - setup: - def api = apiWithVersion(agentVersion) - def writer = DDAgentWriter.builder() - .agentApi(api) - .traceBufferSize(8) - .monitoring(monitoring) - .flushIntervalMilliseconds(-1) - .build() - writer.start() - - when: - writer.flush() - - then: - 0 * _ - - cleanup: - writer.close() - - where: - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.5/traces"] - } - - def "test happy path"() { - setup: - def api = Mock(DDAgentApi) - def discovery = Mock(DDAgentFeaturesDiscovery) - def writer = DDAgentWriter.builder() - .featureDiscovery(discovery) - .agentApi(api) - .traceBufferSize(1024) - .monitoring(monitoring) - .flushIntervalMilliseconds(-1) - .build() - writer.start() - def trace = [dummyTracer.buildSpan("datadog", "fakeOperation").start()] - - when: - writer.write(trace) - writer.write(trace) - writer.flush() - - then: - 2 * discovery.getTraceEndpoint() >> agentVersion - 1 * api.sendSerializedTraces({ it.traceCount() == 2 }) >> RemoteApi.Response.success(200) - 0 * _ - - cleanup: - writer.close() - - where: - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.5/traces"] - } - - def "test flood of traces"() { - setup: - def api = Mock(DDAgentApi) - def discovery = Mock(DDAgentFeaturesDiscovery) - def writer = DDAgentWriter.builder() - .featureDiscovery(discovery) - .agentApi(api) - .traceBufferSize(bufferSize) - .monitoring(monitoring) - .flushIntervalMilliseconds(-1) - .build() - writer.start() - def trace = [dummyTracer.buildSpan("datadog", "fakeOperation").start()] - - when: - (1..traceCount).each { - writer.write(trace) - } - writer.flush() - - then: - 2 * discovery.getTraceEndpoint() >> agentVersion - 1 * api.sendSerializedTraces({ it.traceCount() <= traceCount }) >> RemoteApi.Response.success(200) - 0 * _ - - cleanup: - writer.close() - - where: - bufferSize = 1024 - traceCount = 100 // Shouldn't trigger payload, but bigger than the disruptor size. - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.5/traces"] - } - - def "test flush by time"() { - setup: - def healthMetrics = Mock(HealthMetrics) - def api = Mock(DDAgentApi) - def discovery = Mock(DDAgentFeaturesDiscovery) - def writer = DDAgentWriter.builder() - .featureDiscovery(discovery) - .agentApi(api) - .healthMetrics(healthMetrics) - .monitoring(monitoring) - .flushIntervalMilliseconds(1000) - .build() - writer.start() - def span = dummyTracer.buildSpan("datadog", "fakeOperation").start() - def trace = (1..10).collect { span } - - when: - (1..5).each { - writer.write(trace) - } - phaser.awaitAdvanceInterruptibly(phaser.arriveAndDeregister()) - - then: - 2 * discovery.getTraceEndpoint() >> agentVersion - 1 * healthMetrics.onSerialize(_) - 1 * api.sendSerializedTraces({ it.traceCount() == 5 }) >> RemoteApi.Response.success(200) - _ * healthMetrics.onPublish(_, _) - 1 * healthMetrics.onSend(_, _, _) >> { - phaser.arrive() - } - 0 * _ - - cleanup: - writer.close() - - where: - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.5/traces"] - } - - @Timeout(30) - def "test default buffer size for #agentVersion"() { - setup: - // disable process tags since they are only written on the first span and it will break the trace size estimation - injectSysConfig(EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED, "false") - ProcessTags.reset() - def api = Mock(DDAgentApi) - def discovery = Mock(DDAgentFeaturesDiscovery) - def writer = DDAgentWriter.builder() - .featureDiscovery(discovery) - .agentApi(api) - .traceBufferSize(BUFFER_SIZE) - .prioritization(ENSURE_TRACE) - .monitoring(monitoring) - .flushIntervalMilliseconds(-1) - .build() - writer.start() - - when: - def mapper = agentVersion.equals("v0.5/traces") ? new TraceMapperV0_5() : new TraceMapperV0_4() - int traceSize = calculateSize(minimalTrace, mapper) - int maxedPayloadTraceCount = ((int) ((mapper.messageBufferSize()) / traceSize)) - (0..maxedPayloadTraceCount).each { - writer.write(minimalTrace) - } - writer.flush() - - then: - 2 * discovery.getTraceEndpoint() >> agentVersion - 1 * api.sendSerializedTraces({ it.traceCount() == maxedPayloadTraceCount }) >> RemoteApi.Response.success(200) - 1 * api.sendSerializedTraces({ it.traceCount() == 1 }) >> RemoteApi.Response.success(200) - 0 * _ - - cleanup: - injectSysConfig(EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED, "true") - ProcessTags.reset() - writer.close() - - where: - minimalTrace = createMinimalTrace() - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.5/traces"] - } - - def "check that there are no interactions after close"() { - setup: - def healthMetrics = Mock(HealthMetrics) - def api = Mock(DDAgentApi) - def discovery = Mock(DDAgentFeaturesDiscovery) - def writer = DDAgentWriter.builder() - .featureDiscovery(discovery) - .agentApi(api) - .healthMetrics(healthMetrics) - .monitoring(monitoring) - .build() - writer.start() - - when: - writer.close() - writer.write([]) - writer.flush() - - then: - // this will be checked during flushing - 1 * healthMetrics.onFailedPublish(_,_) - 1 * healthMetrics.onFlush(_) - 1 * healthMetrics.onShutdown(_) - 1 * healthMetrics.close() - 0 * _ - - cleanup: - writer.close() - - where: - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.5/traces"] - } - - def createMinimalContext() { - def tracer = Stub(CoreTracer) - def trace = Stub(PendingTrace) - trace.mapServiceName(_) >> { String serviceName -> serviceName } - trace.getTracer() >> tracer - return new DDSpanContext( - DDTraceId.ONE, - 1, - DDSpanId.ZERO, - "", - "", - "", - "", - PrioritySampling.UNSET, - "", - [:], - false, - "", - 0, - trace, - null, - null, - NoopPathwayContext.INSTANCE, - false, - PropagationTags.factory().empty()) - } - - def createMinimalTrace() { - def context = createMinimalContext() - def minimalSpan = new DDSpan("test", 0, context, null) - context.getTraceCollector().getRootSpan() >> minimalSpan - def minimalTrace = [minimalSpan] - - return minimalTrace - } - - def "monitor happy path"() { - setup: - def healthMetrics = Mock(HealthMetrics) - def minimalTrace = createMinimalTrace() - - // DQH -- need to set-up a dummy agent for the final send callback to work - def agent = httpServer { - handlers { - put(agentVersion) { - response.status(200).send() - } - } - } - def agentUrl = HttpUrl.get(agent.address) - def client = OkHttpUtils.buildHttpClient(agentUrl, 1000) - def discovery = new DDAgentFeaturesDiscovery(client, monitoring, agentUrl, V0_5, true, false) - def api = new DDAgentApi(client, agentUrl, discovery, monitoring, true) - def writer = DDAgentWriter.builder() - .featureDiscovery(discovery) - .agentApi(api) - .monitoring(monitoring) - .healthMetrics(healthMetrics).build() - - when: - writer.start() - - then: - 1 * healthMetrics.onStart(writer.getCapacity()) - - when: - writer.write(minimalTrace) - writer.flush() - - then: - 1 * healthMetrics.onPublish(minimalTrace, _) - 1 * healthMetrics.onSerialize(_) - 1 * healthMetrics.onFlush(false) - 1 * healthMetrics.onSend(1, _, { response -> response.success() && response.status().present && response.status().asInt == 200 }) - - when: - writer.close() - - then: - 1 * healthMetrics.onShutdown(true) - - cleanup: - agent.close() - - where: - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.5/traces"] - } - - def "monitor agent returns error"() { - setup: - def healthMetrics = Mock(HealthMetrics) - def minimalTrace = createMinimalTrace() - - // DQH -- need to set-up a dummy agent for the final send callback to work - def first = true - def agent = httpServer { - handlers { - put(agentVersion) { - // DQH - DDApi sniffs for end point existence, so respond with 200 the first time - if (first) { - response.status(200).send() - first = false - } else { - response.status(500).send() - } - } - } - } - def agentUrl = HttpUrl.get(agent.address) - def client = OkHttpUtils.buildHttpClient(agentUrl, 1000) - def discovery = new DDAgentFeaturesDiscovery(client, monitoring, agentUrl, V0_5, true, false) - def api = new DDAgentApi(client, agentUrl, discovery, monitoring, true) - def writer = DDAgentWriter.builder() - .featureDiscovery(discovery) - .agentApi(api) - .monitoring(monitoring) - .healthMetrics(healthMetrics).build() - - when: - writer.start() - - then: - 1 * healthMetrics.onStart(writer.getCapacity()) - - when: - writer.write(minimalTrace) - writer.flush() - - then: - 1 * healthMetrics.onPublish(minimalTrace, _) - 1 * healthMetrics.onSerialize(_) - 1 * healthMetrics.onFlush(false) - 1 * healthMetrics.onFailedSend(1, _, { response -> !response.success() && response.status().present && response.status().asInt == 500 }) - - when: - writer.close() - - then: - 1 * healthMetrics.onShutdown(true) - - cleanup: - agent.close() - - where: - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.5/traces"] - } - - def "unreachable agent test"() { - setup: - def healthMetrics = Mock(HealthMetrics) - def minimalTrace = createMinimalTrace() - def version = agentVersion - def discovery = Stub(DDAgentFeaturesDiscovery) { - it.getTraceEndpoint() >> version - } - def api = Mock(DDAgentApi) { - it.sendSerializedTraces(_) >> { - // simulating a communication failure to a server - return RemoteApi.Response.failed(new IOException("comm error")) - } - } - - def writer = DDAgentWriter.builder() - .featureDiscovery(discovery) - .agentApi(api) - .monitoring(monitoring) - .healthMetrics(healthMetrics).build() - - when: - writer.start() - - then: - 1 * healthMetrics.onStart(writer.getCapacity()) - - when: - writer.write(minimalTrace) - writer.flush() - - then: - // if we know there's no agent, we'll drop the traces before serialising them - // but we also know that there's nowhere to send health metrics to - 1 * healthMetrics.onPublish(_, _) - 1 * healthMetrics.onFlush(false) - - when: - writer.close() - - then: - 1 * healthMetrics.onShutdown(true) - - where: - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.5/traces"] - } - - @Flaky("If execution is too slow, the http client timeout may trigger") - def "slow response test"() { - def numWritten = 0 - def numFlushes = new AtomicInteger(0) - def numPublished = new AtomicInteger(0) - def numFailedPublish = new AtomicInteger(0) - def numRequests = new AtomicInteger(0) - def numFailedRequests = new AtomicInteger(0) - - def responseSemaphore = new Semaphore(1) - - setup: - def version = agentVersion - - // Need to set-up a dummy agent for the final send callback to work - def agent = httpServer { - handlers { - put(version) { - responseSemaphore.acquire() - try { - response.status(200).send() - } finally { - responseSemaphore.release() - } - } - } - } - - // This test focuses just on failed publish, so not verifying every callback - def healthMetrics = Stub(HealthMetrics) { - onPublish(_, _) >> { - numPublished.incrementAndGet() - } - onFailedPublish(_,_) >> { - numFailedPublish.incrementAndGet() - } - onFlush(_) >> { - numFlushes.incrementAndGet() - } - onSend(_, _, _) >> { - numRequests.incrementAndGet() - } - onFailedSend(_, _, _) >> { - numFailedRequests.incrementAndGet() - } - } - - def writer = DDAgentWriter.builder() - .traceAgentProtocolVersion(V0_5) - .traceAgentPort(agent.address.port) - .monitoring(monitoring) - .healthMetrics(healthMetrics) - .traceBufferSize(bufferSize).build() - writer.start() - - // gate responses - responseSemaphore.acquire() - - when: - // write a single trace and flush - // with responseSemaphore held, the response is blocked but may still time out - writer.write(minimalTrace) - numWritten += 1 - - // sanity check coordination mechanism of test - // release to allow response to be generated - responseSemaphore.release() - writer.flush() - - // reacquire semaphore to stall further responses - responseSemaphore.acquire() - - then: - numFailedPublish.get() == 0 - numPublished.get() == numWritten - numPublished.get() + numFailedPublish.get() == numWritten - numFlushes.get() == 1 - - when: - // send many traces to fill the sender queue... - // loop until outstanding requests > finished requests - while (writer.traceProcessingWorker.getRemainingCapacity() > 0 || numFailedPublish.get() == 0) { - writer.write(minimalTrace) - numWritten += 1 - } - - then: - numFailedPublish.get() > 0 - numPublished.get() + numFailedPublish.get() == numWritten - - when: - - // with both disruptor & queue full, should reject everything - def expectedRejects = 100 - (1..expectedRejects).each { - writer.write(minimalTrace) - numWritten += 1 - } - - then: - numPublished.get() + numFailedPublish.get() == numWritten - - cleanup: - responseSemaphore.release() - - writer.close() - agent.close() - - where: - bufferSize = 16 - minimalTrace = createMinimalTrace() - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.5/traces"] - } - - def "multi threaded"() { - def numPublished = new AtomicInteger(0) - def numFailedPublish = new AtomicInteger(0) - def numRepSent = new AtomicInteger(0) - - setup: - def minimalTrace = createMinimalTrace() - def version = agentVersion - - // Need to set-up a dummy agent for the final send callback to work - def agent = httpServer { - handlers { - put(version) { - response.status(200).send() - } - } - } - - // This test focuses just on failed publish, so not verifying every callback - def healthMetrics = Stub(HealthMetrics) { - onPublish(_, _) >> { - numPublished.incrementAndGet() - } - onFailedPublish(_,_) >> { - numFailedPublish.incrementAndGet() - } - onSend(_, _, _) >> { repCount, sizeInBytes, response -> - numRepSent.addAndGet(repCount) - } - } - - def writer = DDAgentWriter.builder() - .traceAgentProtocolVersion(V0_5) - .traceAgentPort(agent.address.port) - .monitoring(monitoring) - .healthMetrics(healthMetrics).build() - writer.start() - - when: - def producer = { - (1..100).each { - writer.write(minimalTrace) - } - } as Runnable - - def t1 = new Thread(producer) - t1.start() - - def t2 = new Thread(producer) - t2.start() - - t1.join() - t2.join() - - writer.flush() - - then: - conditions.eventually { - def totalTraces = 100 + 100 - assert numPublished.get() == totalTraces - assert numRepSent.get() == totalTraces - } - - cleanup: - writer.close() - agent.close() - - where: - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.5/traces"] - } - - def "statsd success"() { - def numTracesAccepted = new AtomicInteger(0) - def numRequests = new AtomicInteger(0) - def numResponses = new AtomicInteger(0) - - setup: - def minimalTrace = createMinimalTrace() - def version = agentVersion - - // Need to set-up a dummy agent for the final send callback to work - def agent = httpServer { - handlers { - put(version) { - response.status(200).send() - } - } - } - - def healthMetrics = Stub(HealthMetrics) - healthMetrics.onPublish(_, _) >> { - numTracesAccepted.incrementAndGet() - } - healthMetrics.onSend(_, _, _) >> { - numRequests.incrementAndGet() - numResponses.incrementAndGet() - } - def writer = DDAgentWriter.builder() - .agentHost(agent.address.host) - .traceAgentProtocolVersion(V0_5) - .traceAgentPort(agent.address.port) - .monitoring(monitoring) - .healthMetrics(healthMetrics).build() - writer.start() - - when: - writer.write(minimalTrace) - writer.flush() - - then: - numTracesAccepted.get() == 1 - numRequests.get() == 1 - numResponses.get() == 1 - - cleanup: - agent.close() - writer.close() - - where: - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.5/traces"] - } - - def "statsd comm failure"() { - setup: - def minimalTrace = createMinimalTrace() - - def api = apiWithVersion(agentVersion) - api.sendSerializedTraces(_) >> RemoteApi.Response.failed(new IOException("comm error")) - - def latch = new CountDownLatch(2) - def statsd = Mock(StatsDClient) - def healthMetrics = new TracerHealthMetrics(statsd, 100, TimeUnit.MILLISECONDS) - def writer = DDAgentWriter.builder() - .traceAgentProtocolVersion(V0_5) - .agentApi(api).monitoring(monitoring) - .healthMetrics(healthMetrics).build() - healthMetrics.start() - writer.start() - - when: - writer.write(minimalTrace) - writer.flush() - latch.await(10, TimeUnit.SECONDS) - - then: - 1 * statsd.count("api.requests.total", 1, _) >> { - latch.countDown() - } - 0 * statsd.incrementCounter("api.responses.total", _) - 1 * statsd.count("api.errors.total", 1, _) >> { - latch.countDown() - } - - cleanup: - writer.close() - healthMetrics.close() - - where: - agentVersion << ["v0.3/traces", "v0.4/traces", "v0.5/traces"] - } - - static int calculateSize(List trace, Mapper> mapper) { - AtomicInteger size = new AtomicInteger() - def packer = new MsgPackWriter(new FlushingBuffer(1024, new ByteBufferConsumer() { - @Override - void accept(int messageCount, ByteBuffer buffer) { - size.set(buffer.limit() - buffer.position()) - } - })) - packer.format(trace, mapper) - packer.flush() - return size.get() - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentWriterTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentWriterTest.groovy deleted file mode 100644 index 965f3a88d5e..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentWriterTest.groovy +++ /dev/null @@ -1,222 +0,0 @@ -package datadog.trace.common.writer - -import datadog.communication.ddagent.DDAgentFeaturesDiscovery -import datadog.trace.api.DDSpanId -import datadog.trace.api.DDTraceId -import datadog.trace.api.sampling.PrioritySampling -import datadog.trace.api.datastreams.NoopPathwayContext -import datadog.trace.common.writer.ddagent.DDAgentApi -import datadog.trace.common.writer.ddagent.DDAgentMapperDiscovery -import datadog.metrics.api.statsd.StatsDClient -import datadog.metrics.impl.MonitoringImpl -import datadog.trace.core.CoreTracer -import datadog.trace.core.DDSpan -import datadog.trace.core.DDSpanContext -import datadog.trace.core.PendingTrace -import datadog.trace.core.monitor.HealthMetrics -import datadog.trace.core.propagation.PropagationTags -import datadog.trace.core.test.DDCoreSpecification -import spock.lang.Subject - -import java.util.concurrent.TimeUnit - -import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.DROPPED_BUFFER_OVERFLOW -import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.DROPPED_BY_POLICY -import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.ENQUEUED_FOR_SERIALIZATION -import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.ENQUEUED_FOR_SINGLE_SPAN_SAMPLING - -class DDAgentWriterTest extends DDCoreSpecification { - - def monitor = Mock(HealthMetrics) - def worker = Mock(TraceProcessingWorker) - def discovery = Mock(DDAgentFeaturesDiscovery) - def api = Mock(DDAgentApi) - def monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS) - def dispatcher = new PayloadDispatcherImpl(new DDAgentMapperDiscovery(discovery), api, monitor, monitoring) - - @Subject - def writer = new DDAgentWriter(worker, dispatcher, monitor, 1, TimeUnit.SECONDS, false) - - // Only used to create spans - def dummyTracer = tracerBuilder().writer(new ListWriter()).build() - - def cleanup() { - writer.close() - dummyTracer.close() - } - - def "test writer builder"() { - when: - def writer = DDAgentWriter.builder().build() - - then: - writer != null - } - - def "test writer.start"() { - when: - writer.start() - - then: - 1 * monitor.start() - 1 * worker.start() - 1 * worker.getCapacity() >> capacity - 1 * monitor.onStart(capacity) - 0 * _ - - where: - capacity = 5 - } - - def "test writer.start closed"() { - setup: - writer.close() - - when: - writer.start() - - then: - 0 * _ - } - - def "test writer.flush"() { - when: - writer.flush() - - then: - 1 * worker.flush(1, TimeUnit.SECONDS) >> true - 1 * monitor.onFlush(false) - 0 * _ - - when: - writer.flush() - - then: - 1 * worker.flush(1, TimeUnit.SECONDS) >> false - 0 * _ - } - - def "test writer.flush closed"() { - setup: - writer.close() - - when: - writer.flush() - - then: - 0 * _ - } - - def "test writer.write publish succeeds"() { - setup: - def trace = [dummyTracer.buildSpan("datadog", "fakeOperation").start()] - - when: "publish succeeds" - writer.write(trace) - - then: "monitor is notified of successful publication" - 1 * worker.publish(_, _, trace) >> ENQUEUED_FOR_SERIALIZATION - 1 * monitor.onPublish(trace, _) - 0 * _ - } - - def "test writer.write publish for single span sampling"() { - setup: - def trace = [dummyTracer.buildSpan("datadog", "fakeOperation").start()] - - when: "publish succeeds" - writer.write(trace) - - then: "monitor is notified of successful publication" - 1 * worker.publish(_, _, trace) >> ENQUEUED_FOR_SINGLE_SPAN_SAMPLING - // shouldn't call monitor.onPublish - 0 * _ - } - - def "test writer.write publish fails"() { - setup: - def trace = [dummyTracer.buildSpan("datadog", "fakeOperation").start()] - - when: "publish fails" - writer.write(trace) - - then: "monitor is notified of unsuccessful publication" - 1 * worker.publish(_, _, trace) >> publishResult - 1 * monitor.onFailedPublish(_,_) - 0 * _ - - where: - publishResult << [DROPPED_BUFFER_OVERFLOW, DROPPED_BY_POLICY] - } - - def "empty traces should be reported as failures"() { - when: "trace is empty" - writer.write([]) - - then: "monitor is notified of unsuccessful publication" - 1 * monitor.onFailedPublish(_,_) - 0 * _ - } - - def "test writer.write closed"() { - setup: - writer.close() - def trace = [dummyTracer.buildSpan("datadog", "fakeOperation").start()] - - when: - writer.write(trace) - - then: - 1 * monitor.onFailedPublish(_,_) - 0 * _ - } - - def "dropped trace is counted"() { - setup: - def worker = Mock(TraceProcessingWorker) - def monitor = Stub(HealthMetrics) - def dispatcher = Mock(PayloadDispatcherImpl) - def writer = new DDAgentWriter(worker, dispatcher, monitor, 1, TimeUnit.SECONDS, false) - def p0 = newSpan() - p0.setSamplingPriority(PrioritySampling.SAMPLER_DROP) - def trace = [p0, newSpan()] - - when: - writer.write(trace) - - then: - 1 * worker.publish(trace[0], PrioritySampling.SAMPLER_DROP, trace) >> publishResult - 1 * dispatcher.onDroppedTrace(trace.size()) - - where: - publishResult << [DROPPED_BY_POLICY, DROPPED_BUFFER_OVERFLOW] - } - - def newSpan() { - CoreTracer tracer = Stub(CoreTracer) - PendingTrace trace = Stub(PendingTrace) - trace.mapServiceName(_) >> { String serviceName -> serviceName } - trace.getTracer() >> tracer - def context = new DDSpanContext( - DDTraceId.ONE, - 1, - DDSpanId.ZERO, - null, - "", - "", - "", - PrioritySampling.UNSET, - "", - [:], - false, - "", - 0, - trace, - null, - null, - NoopPathwayContext.INSTANCE, - false, - PropagationTags.factory().empty()) - return new DDSpan("test", 0, context, null) - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDIntakeWriterCombinedTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDIntakeWriterCombinedTest.groovy deleted file mode 100644 index 966d4e91ecb..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDIntakeWriterCombinedTest.groovy +++ /dev/null @@ -1,769 +0,0 @@ -package datadog.trace.common.writer - -import datadog.communication.http.OkHttpUtils -import datadog.communication.serialization.ByteBufferConsumer -import datadog.communication.serialization.FlushingBuffer -import datadog.communication.serialization.msgpack.MsgPackWriter -import datadog.trace.api.DDSpanId -import datadog.trace.api.DDTraceId -import datadog.trace.api.civisibility.CiVisibilityWellKnownTags -import datadog.trace.api.intake.TrackType -import datadog.trace.api.sampling.PrioritySampling -import datadog.trace.api.datastreams.NoopPathwayContext -import datadog.metrics.impl.MonitoringImpl -import datadog.metrics.api.statsd.StatsDClient -import datadog.trace.common.writer.ddintake.DDIntakeApi -import datadog.trace.common.writer.ddintake.DDIntakeMapperDiscovery -import datadog.trace.core.CoreTracer -import datadog.trace.core.DDSpan -import datadog.trace.core.DDSpanContext -import datadog.trace.core.PendingTrace -import datadog.trace.core.monitor.HealthMetrics -import datadog.trace.core.monitor.TracerHealthMetrics -import datadog.trace.core.propagation.PropagationTags -import datadog.trace.core.test.DDCoreSpecification -import datadog.trace.test.util.Flaky -import okhttp3.HttpUrl -import spock.lang.Shared -import spock.lang.Timeout -import spock.util.concurrent.PollingConditions - -import java.nio.ByteBuffer -import java.util.concurrent.CountDownLatch -import java.util.concurrent.Phaser -import java.util.concurrent.Semaphore -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicInteger - -import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer -import static datadog.trace.common.writer.DDIntakeWriter.BUFFER_SIZE -import static datadog.trace.common.writer.ddagent.Prioritization.ENSURE_TRACE - -@Timeout(10) -class DDIntakeWriterCombinedTest extends DDCoreSpecification { - - @Shared - def wellKnownTags = new CiVisibilityWellKnownTags( - "my-runtime-id", "my-env", "my-language", - "my-runtime-name", "my-runtime-version", "my-runtime-vendor", - "my-os-arch", "my-os-platform", "my-os-version", "false") - - def conditions = new PollingConditions(timeout: 5, initialDelay: 0, factor: 1.25) - def monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS) - def phaser = new Phaser() - - // Only used to create spans - def dummyTracer = tracerBuilder().writer(new ListWriter()).build() - - def setup() { - // Register for two threads. - phaser.register() - phaser.register() - } - - def cleanup() { - dummyTracer?.close() - } - - def "no interactions because of initial flush"() { - setup: - def api = Mock(DDIntakeApi) - def writer = DDIntakeWriter.builder() - .addTrack(TrackType.NOOP, api) - .traceBufferSize(8) - .monitoring(monitoring) - .flushIntervalMilliseconds(-1) - .alwaysFlush(false) - .build() - writer.start() - - when: - writer.flush() - - then: - 0 * _ - - cleanup: - writer.close() - } - - def "test happy path"() { - setup: - def api = Mock(DDIntakeApi) - def writer = DDIntakeWriter.builder() - .addTrack(trackType, api) - .traceBufferSize(1024) - .monitoring(monitoring) - .flushIntervalMilliseconds(-1) - .alwaysFlush(false) - .build() - writer.start() - def trace = [dummyTracer.buildSpan("datadog", "fakeOperation").start()] - - when: - writer.write(trace) - writer.write(trace) - writer.flush() - - then: - 1 * api.sendSerializedTraces({ it.traceCount() == 2 }) >> RemoteApi.Response.success(200) - 0 * _ - - cleanup: - writer.close() - - where: - trackType << [TrackType.CITESTCYCLE] - } - - def "test flood of traces"() { - setup: - def api = Mock(DDIntakeApi) - def writer = DDIntakeWriter.builder() - .addTrack(trackType, api) - .traceBufferSize(1024) - .monitoring(monitoring) - .flushIntervalMilliseconds(-1) - .alwaysFlush(false) - .build() - writer.start() - def trace = [dummyTracer.buildSpan("datadog", "fakeOperation").start()] - - when: - (1..traceCount).each { - writer.write(trace) - } - writer.flush() - - then: - 1 * api.sendSerializedTraces({ it.traceCount() <= traceCount }) >> RemoteApi.Response.success(200) - 0 * _ - - cleanup: - writer.close() - - where: - bufferSize = 1024 - traceCount = 100 // Shouldn't trigger payload, but bigger than the disruptor size. - trackType << [TrackType.CITESTCYCLE] - } - - def "test flush by time"() { - setup: - def healthMetrics = Mock(HealthMetrics) - def api = Mock(DDIntakeApi) - def writer = DDIntakeWriter.builder() - .addTrack(trackType, api) - .healthMetrics(healthMetrics) - .monitoring(monitoring) - .flushIntervalMilliseconds(1000) - .alwaysFlush(false) - .build() - writer.start() - def span = dummyTracer.buildSpan("datadog", "fakeOperation").start() - def trace = (1..10).collect { span } - - when: - (1..5).each { - writer.write(trace) - } - phaser.awaitAdvanceInterruptibly(phaser.arriveAndDeregister()) - - then: - 1 * healthMetrics.onSerialize(_) - 1 * api.sendSerializedTraces({ it.traceCount() == 5 }) >> RemoteApi.Response.success(200) - _ * healthMetrics.onPublish(_, _) - 1 * healthMetrics.onSend(_, _, _) >> { - phaser.arrive() - } - 0 * _ - - cleanup: - writer.close() - - where: - trackType << [TrackType.CITESTCYCLE] - } - - @Timeout(30) - def "test default buffer size for #trackType"() { - setup: - def api = Mock(DDIntakeApi) - def writer = DDIntakeWriter.builder() - .addTrack(trackType, api) - .wellKnownTags(wellKnownTags) - .traceBufferSize(BUFFER_SIZE) - .prioritization(ENSURE_TRACE) - .monitoring(monitoring) - .flushIntervalMilliseconds(-1) - .alwaysFlush(false) - .build() - writer.start() - - when: - def discovery = new DDIntakeMapperDiscovery(trackType, wellKnownTags, false) - discovery.discover() - def mapper = (RemoteMapper) discovery.mapper - def traceSize = calculateSize(minimalTrace, mapper) - int maxedPayloadTraceCount = ((int) ((mapper.messageBufferSize()) / traceSize)) - (0..maxedPayloadTraceCount).each { - writer.write(minimalTrace) - } - writer.flush() - - then: - 1 * api.sendSerializedTraces({ it.traceCount() == maxedPayloadTraceCount }) >> RemoteApi.Response.success(200) - 1 * api.sendSerializedTraces({ it.traceCount() == 1 }) >> RemoteApi.Response.success(200) - 0 * _ - - cleanup: - writer.close() - - where: - minimalTrace = createMinimalTrace() - trackType << [TrackType.CITESTCYCLE] - } - - def "check that there are no interactions after close"() { - setup: - def api = Mock(DDIntakeApi) - def healthMetrics = Mock(HealthMetrics) - def writer = DDIntakeWriter.builder() - .addTrack(trackType, api) - .healthMetrics(healthMetrics) - .monitoring(monitoring) - .alwaysFlush(false) - .build() - writer.start() - - when: - writer.close() - writer.write([]) - writer.flush() - - then: - // this will be checked during flushing - 1 * healthMetrics.onFailedPublish(_,_) - 1 * healthMetrics.onFlush(_) - 1 * healthMetrics.onShutdown(_) - 1 * healthMetrics.close() - 0 * _ - - cleanup: - writer.close() - - where: - trackType << [TrackType.CITESTCYCLE] - } - - def "monitor happy path"() { - setup: - def healthMetrics = Mock(HealthMetrics) - def minimalTrace = createMinimalTrace() - def path = buildIntakePath(trackType, apiVersion) - def intake = httpServer { - handlers { - post(path) { - response.status(200).send() - } - } - } - def hostUrl = HttpUrl.get(intake.address) - def client = OkHttpUtils.buildHttpClient(hostUrl, 1000) - def api = DDIntakeApi.builder() - .hostUrl(hostUrl) - .httpClient(client) - .apiKey("my-api-key") - .trackType(trackType) - .build() - def writer = DDIntakeWriter.builder() - .addTrack(trackType, api) - .healthMetrics(healthMetrics) - .monitoring(monitoring) - .alwaysFlush(false) - .build() - - when: - writer.start() - - then: - 1 * healthMetrics.onStart(writer.getCapacity()) - - when: - writer.write(minimalTrace) - writer.flush() - - then: - 1 * healthMetrics.onPublish(minimalTrace, _) - 1 * healthMetrics.onSerialize(_) - 1 * healthMetrics.onFlush(false) - 1 * healthMetrics.onSend(1, _, { response -> response.success() && response.status().present && response.status().asInt == 200 }) - - when: - writer.close() - - then: - 1 * healthMetrics.onShutdown(true) - - cleanup: - intake.close() - - where: - trackType | apiVersion - TrackType.CITESTCYCLE | "v2" - } - - def "monitor intake returns error"() { - setup: - def healthMetrics = Mock(HealthMetrics) - def minimalTrace = createMinimalTrace() - - def path = buildIntakePath(trackType, apiVersion) - def intake = httpServer { - handlers { - post(path) { - response.status(500).send() - } - } - } - def hostUrl = HttpUrl.get(intake.address) - def client = OkHttpUtils.buildHttpClient(hostUrl, 1000) - def api = DDIntakeApi.builder() - .hostUrl(hostUrl) - .httpClient(client) - .apiKey("my-api-key") - .trackType(trackType) - .build() - def writer = DDIntakeWriter.builder() - .addTrack(trackType, api) - .healthMetrics(healthMetrics) - .monitoring(monitoring) - .alwaysFlush(false) - .build() - - when: - writer.start() - - then: - 1 * healthMetrics.onStart(writer.getCapacity()) - - when: - writer.write(minimalTrace) - writer.flush() - - then: - 1 * healthMetrics.onPublish(minimalTrace, _) - 1 * healthMetrics.onSerialize(_) - 1 * healthMetrics.onFlush(false) - 1 * healthMetrics.onFailedSend(1, _, { response -> !response.success() && response.status().present && response.status().asInt == 500 }) - - when: - writer.close() - - then: - 1 * healthMetrics.onShutdown(true) - - cleanup: - intake.close() - - where: - trackType | apiVersion - TrackType.CITESTCYCLE | "v2" - } - - def "unreachable intake test"() { - setup: - def healthMetrics = Mock(HealthMetrics) - def minimalTrace = createMinimalTrace() - - def api = Mock(DDIntakeApi) { - it.sendSerializedTraces(_) >> { - // simulating a communication failure to a server - return RemoteApi.Response.failed(new IOException("comm error")) - } - } - - def writer = DDIntakeWriter.builder() - .addTrack(trackType, api) - .monitoring(monitoring) - .healthMetrics(healthMetrics) - .alwaysFlush(false) - .build() - - when: - writer.start() - - then: - 1 * healthMetrics.onStart(writer.getCapacity()) - - when: - writer.write(minimalTrace) - writer.flush() - - then: - // if we know there's no agent, we'll drop the traces before serialising them - // but we also know that there's nowhere to send health metrics to - 1 * healthMetrics.onPublish(_, _) - 1 * healthMetrics.onFlush(false) - - when: - writer.close() - - then: - 1 * healthMetrics.onShutdown(true) - - where: - trackType << [TrackType.CITESTCYCLE] - } - - @Flaky - // if execution is too slow, the http client timeout may trigger. - def "slow response test"() { - def numWritten = 0 - def numFlushes = new AtomicInteger(0) - def numPublished = new AtomicInteger(0) - def numFailedPublish = new AtomicInteger(0) - def numRequests = new AtomicInteger(0) - def numFailedRequests = new AtomicInteger(0) - - def responseSemaphore = new Semaphore(1) - - setup: - def path = buildIntakePath(trackType, apiVersion) - def intake = httpServer { - handlers { - post(path) { - responseSemaphore.acquire() - try { - response.status(200).send() - } finally { - responseSemaphore.release() - } - } - } - } - // This test focuses just on failed publish, so not verifying every callback - def healthMetrics = Stub(HealthMetrics) { - onPublish(_, _) >> { - numPublished.incrementAndGet() - } - onFailedPublish(_,_) >> { - numFailedPublish.incrementAndGet() - } - onFlush(_) >> { - numFlushes.incrementAndGet() - } - onSend(_, _, _) >> { - numRequests.incrementAndGet() - } - onFailedSend(_, _, _) >> { - numFailedRequests.incrementAndGet() - } - } - - def hostUrl = HttpUrl.get(intake.address) - def client = OkHttpUtils.buildHttpClient(hostUrl, 1000) - def api = DDIntakeApi.builder() - .hostUrl(hostUrl) - .httpClient(client) - .apiKey("my-api-key") - .trackType(trackType) - .build() - def writer = DDIntakeWriter.builder() - .addTrack(trackType, api) - .healthMetrics(healthMetrics) - .traceBufferSize(bufferSize) - .alwaysFlush(false) - .build() - writer.start() - - // gate responses - responseSemaphore.acquire() - - // sanity check coordination mechanism of test - // release to allow response to be generated - responseSemaphore.release() - writer.flush() - - // reacquire semaphore to stall further responses - responseSemaphore.acquire() - - when: - // write a single trace and flush - // with responseSemaphore held, the response is blocked but may still time out - writer.write(minimalTrace) - numWritten += 1 - - then: - numFailedPublish.get() == 0 - numPublished.get() == numWritten - numPublished.get() + numFailedPublish.get() == numWritten - numFlushes.get() == 1 - - when: - // send many traces to fill the sender queue... - // loop until outstanding requests > finished requests - while (writer.traceProcessingWorker.getRemainingCapacity() > 0 || numFailedPublish.get() == 0) { - writer.write(minimalTrace) - numWritten += 1 - } - - then: - numFailedPublish.get() > 0 - numPublished.get() + numFailedPublish.get() == numWritten - - when: - - // with both disruptor & queue full, should reject everything - def expectedRejects = 100 - (1..expectedRejects).each { - writer.write(minimalTrace) - numWritten += 1 - } - - then: - numPublished.get() + numFailedPublish.get() == numWritten - - cleanup: - responseSemaphore.release() - - writer.close() - intake.close() - - where: - bufferSize = 16 - minimalTrace = createMinimalTrace() - trackType | apiVersion - TrackType.CITESTCYCLE | "v2" - } - - def "multi threaded"() { - def numPublished = new AtomicInteger(0) - def numFailedPublish = new AtomicInteger(0) - def numRepSent = new AtomicInteger(0) - - setup: - def minimalTrace = createMinimalTrace() - def path = buildIntakePath(trackType, apiVersion) - def intake = httpServer { - handlers { - post(path) { - response.status(200).send() - } - } - } - // This test focuses just on failed publish, so not verifying every callback - def healthMetrics = Stub(HealthMetrics) { - onPublish(_, _) >> { - numPublished.incrementAndGet() - } - onFailedPublish(_,_) >> { - numFailedPublish.incrementAndGet() - } - onSend(_, _, _) >> { repCount, sizeInBytes, response -> - numRepSent.addAndGet(repCount) - } - } - - def hostUrl = HttpUrl.get(intake.address) - def client = OkHttpUtils.buildHttpClient(hostUrl, 1000) - def api = DDIntakeApi.builder() - .hostUrl(hostUrl) - .httpClient(client) - .apiKey("my-api-key") - .trackType(trackType) - .build() - def writer = DDIntakeWriter.builder() - .addTrack(trackType, api) - .monitoring(monitoring) - .healthMetrics(healthMetrics) - .alwaysFlush(false) - .build() - writer.start() - - when: - def producer = { - (1..100).each { - writer.write(minimalTrace) - } - } as Runnable - - def t1 = new Thread(producer) - t1.start() - - def t2 = new Thread(producer) - t2.start() - - t1.join() - t2.join() - - writer.flush() - - then: - conditions.eventually { - def totalTraces = 100 + 100 - assert numPublished.get() == totalTraces - assert numRepSent.get() == totalTraces - } - - cleanup: - writer.close() - intake.close() - - where: - trackType | apiVersion - TrackType.CITESTCYCLE | "v2" - } - - def "statsd success"() { - def numTracesAccepted = new AtomicInteger(0) - def numRequests = new AtomicInteger(0) - def numResponses = new AtomicInteger(0) - - setup: - def minimalTrace = createMinimalTrace() - def path = buildIntakePath(trackType, apiVersion) - def intake = httpServer { - handlers { - post(path) { - response.status(200).send() - } - } - } - def healthMetrics = Stub(HealthMetrics) - healthMetrics.onPublish(_, _) >> { - numTracesAccepted.incrementAndGet() - } - healthMetrics.onSend(_, _, _) >> { - numRequests.incrementAndGet() - numResponses.incrementAndGet() - } - def hostUrl = HttpUrl.get(intake.address) - def client = OkHttpUtils.buildHttpClient(hostUrl, 1000) - def api = DDIntakeApi.builder() - .hostUrl(hostUrl) - .httpClient(client) - .apiKey("my-api-key") - .trackType(trackType) - .build() - def writer = DDIntakeWriter.builder() - .addTrack(trackType, api) - .monitoring(monitoring) - .healthMetrics(healthMetrics) - .alwaysFlush(false) - .build() - writer.start() - - when: - writer.write(minimalTrace) - writer.flush() - - then: - numTracesAccepted.get() == 1 - numRequests.get() == 1 - numResponses.get() == 1 - - cleanup: - intake.close() - writer.close() - - where: - trackType | apiVersion - TrackType.CITESTCYCLE | "v2" - } - - def "statsd comm failure"() { - setup: - def minimalTrace = createMinimalTrace() - - def api = Mock(DDIntakeApi) - api.sendSerializedTraces(_) >> RemoteApi.Response.failed(new IOException("comm error")) - - def latch = new CountDownLatch(2) - def statsd = Mock(StatsDClient) - def healthMetrics = new TracerHealthMetrics(statsd, 100, TimeUnit.MILLISECONDS) - def writer = DDIntakeWriter.builder() - .addTrack(trackType, api) - .monitoring(monitoring) - .healthMetrics(healthMetrics) - .alwaysFlush(false) - .build() - healthMetrics.start() - writer.start() - - when: - writer.write(minimalTrace) - writer.flush() - latch.await(10, TimeUnit.SECONDS) - - then: - 1 * statsd.count("api.requests.total", 1, _) >> { - latch.countDown() - } - 0 * statsd.incrementCounter("api.responses.total", _) - 1 * statsd.count("api.errors.total", 1, _) >> { - latch.countDown() - } - - cleanup: - writer.close() - healthMetrics.close() - - where: - trackType | apiVersion - TrackType.CITESTCYCLE | "v2" - } - - def createMinimalContext() { - def tracer = Stub(CoreTracer) - def trace = Stub(PendingTrace) - trace.mapServiceName(_) >> { String serviceName -> serviceName } - trace.getTracer() >> tracer - return new DDSpanContext( - DDTraceId.ONE, - 1, - DDSpanId.ZERO, - "", - "", - "", - "", - PrioritySampling.UNSET, - "", - [:], - false, - "", - 0, - trace, - null, - null, - NoopPathwayContext.INSTANCE, - false, - PropagationTags.factory().empty()) - } - - def createMinimalTrace() { - def context = createMinimalContext() - def minimalSpan = new DDSpan("test", 0, context, null) - context.getTraceCollector().getRootSpan() >> minimalSpan - def minimalTrace = [minimalSpan] - - return minimalTrace - } - - static int calculateSize(List trace, RemoteMapper mapper) { - AtomicInteger size = new AtomicInteger() - def packer = new MsgPackWriter(new FlushingBuffer(mapper.messageBufferSize(), new ByteBufferConsumer() { - @Override - void accept(int messageCount, ByteBuffer buffer) { - size.set(buffer.limit() - buffer.position()) - } - })) - packer.format(trace, mapper) - packer.flush() - return size.get() - } - - def buildIntakePath(TrackType trackType, String apiVersion) { - return String.format("/api/%s/%s", apiVersion, trackType.name().toLowerCase()) - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDIntakeWriterTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDIntakeWriterTest.groovy deleted file mode 100644 index e0a347841a1..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDIntakeWriterTest.groovy +++ /dev/null @@ -1,215 +0,0 @@ -package datadog.trace.common.writer - -import datadog.communication.ddagent.DDAgentFeaturesDiscovery -import datadog.trace.api.DDSpanId -import datadog.trace.api.DDTraceId -import datadog.trace.api.intake.TrackType -import datadog.trace.api.sampling.PrioritySampling -import datadog.trace.api.datastreams.NoopPathwayContext -import datadog.metrics.impl.MonitoringImpl -import datadog.metrics.api.statsd.StatsDClient -import datadog.trace.common.writer.ddagent.DDAgentApi -import datadog.trace.common.writer.ddagent.DDAgentMapperDiscovery -import datadog.trace.core.CoreTracer -import datadog.trace.core.DDSpan -import datadog.trace.core.DDSpanContext -import datadog.trace.core.PendingTrace -import datadog.trace.core.monitor.HealthMetrics -import datadog.trace.core.propagation.PropagationTags -import datadog.trace.core.test.DDCoreSpecification -import spock.lang.Subject - -import java.util.concurrent.TimeUnit - -import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.DROPPED_BUFFER_OVERFLOW -import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.DROPPED_BY_POLICY -import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.ENQUEUED_FOR_SERIALIZATION -import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.ENQUEUED_FOR_SINGLE_SPAN_SAMPLING - -class DDIntakeWriterTest extends DDCoreSpecification { - - def healthMetrics = Mock(HealthMetrics) - def worker = Mock(TraceProcessingWorker) - def discovery = Mock(DDAgentFeaturesDiscovery) - def api = Mock(DDAgentApi) - def monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS) - def dispatcher = new PayloadDispatcherImpl(new DDAgentMapperDiscovery(discovery), api, healthMetrics, monitoring) - - @Subject - def writer = new DDIntakeWriter(worker, dispatcher, healthMetrics, false) - - // Only used to create spans - def dummyTracer = tracerBuilder().writer(new ListWriter()).build() - - def cleanup() { - writer.close() - dummyTracer.close() - } - - def "test writer builder"() { - when: - def writer = DDIntakeWriter.builder().addTrack(TrackType.NOOP, Mock(RemoteApi)).build() - - then: - writer != null - } - - def "test writer.start"() { - when: - writer.start() - - then: - 1 * healthMetrics.start() - 1 * worker.start() - 1 * worker.getCapacity() >> capacity - 1 * healthMetrics.onStart(capacity) - 0 * _ - - where: - capacity = 5 - } - - def "test writer.flush"() { - when: - writer.flush() - - then: - 1 * worker.flush(1, TimeUnit.SECONDS) >> true - 1 * healthMetrics.onFlush(false) - 0 * _ - - when: - writer.flush() - - then: - 1 * worker.flush(1, TimeUnit.SECONDS) >> false - 0 * _ - } - - def "test writer.flush closed"() { - setup: - writer.close() - - when: - writer.flush() - - then: - 0 * _ - } - - def "test writer.write publish succeeds"() { - setup: - def trace = [dummyTracer.buildSpan("datadog", "fakeOperation").start()] - - when: "publish succeeds" - writer.write(trace) - - then: "monitor is notified of successful publication" - 1 * worker.publish(_, _, trace) >> ENQUEUED_FOR_SERIALIZATION - 1 * healthMetrics.onPublish(trace, _) - _ * worker.flush(1, TimeUnit.SECONDS) - 0 * _ - } - - def "test writer.write publish for single span sampling"() { - setup: - def trace = [dummyTracer.buildSpan("datadog", "fakeOperation").start()] - - when: "publish succeeds" - writer.write(trace) - - then: "monitor is notified of successful publication" - 1 * worker.publish(_, _, trace) >> ENQUEUED_FOR_SINGLE_SPAN_SAMPLING - // shouldn't call monitor.onPublish - _ * worker.flush(1, TimeUnit.SECONDS) - 0 * _ - } - - def "test writer.write publish fails"() { - setup: - def trace = [dummyTracer.buildSpan("datadog", "fakeOperation").start()] - - when: "publish fails" - writer.write(trace) - - then: "monitor is notified of unsuccessful publication" - 1 * worker.publish(_, _, trace) >> publishResult - 1 * healthMetrics.onFailedPublish(_,1) - _ * worker.flush(1, TimeUnit.SECONDS) - 0 * _ - - where: - publishResult << [DROPPED_BUFFER_OVERFLOW, DROPPED_BY_POLICY] - } - - def "empty traces should be reported as failures"() { - when: "trace is empty" - writer.write([]) - - then: "monitor is notified of unsuccessful publication" - 1 * healthMetrics.onFailedPublish(_,0) - _ * worker.flush(1, TimeUnit.SECONDS) - 0 * _ - } - - def "test writer.write closed"() { - setup: - writer.close() - def trace = [dummyTracer.buildSpan("datadog", "fakeOperation").start()] - - when: - writer.write(trace) - - then: - 1 * healthMetrics.onFailedPublish(_,1) - _ * worker.flush(1, TimeUnit.SECONDS) - 0 * _ - } - - def "dropped trace is counted"() { - setup: - def dispatcher = Mock(PayloadDispatcherImpl) - def writer = new DDIntakeWriter(worker, dispatcher, healthMetrics, true) - def p0 = newSpan() - p0.setSamplingPriority(PrioritySampling.SAMPLER_DROP) - def trace = [p0, newSpan()] - - when: - writer.write(trace) - - then: - 1 * worker.publish(trace[0], PrioritySampling.SAMPLER_DROP, trace) >> publishResult - 1 * dispatcher.onDroppedTrace(trace.size()) - - where: - publishResult << [DROPPED_BY_POLICY, DROPPED_BUFFER_OVERFLOW] - } - - def newSpan() { - CoreTracer tracer = Stub(CoreTracer) - PendingTrace trace = Stub(PendingTrace) - trace.mapServiceName(_) >> { String serviceName -> serviceName } - trace.getTracer() >> tracer - def context = new DDSpanContext( - DDTraceId.ONE, - 1, - DDSpanId.ZERO, - null, - "", - "", - "", - PrioritySampling.UNSET, - "", - [:], - false, - "", - 0, - trace, - null, - null, - NoopPathwayContext.INSTANCE, - false, - PropagationTags.factory().empty()) - return new DDSpan("test", 0, context, null) - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/MultiWriterTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/MultiWriterTest.groovy deleted file mode 100644 index 1e406ae9bcb..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/MultiWriterTest.groovy +++ /dev/null @@ -1,69 +0,0 @@ -package datadog.trace.common.writer - -import datadog.trace.core.DDSpan -import datadog.trace.test.util.DDSpecification - -class MultiWriterTest extends DDSpecification { - - def "test that multi writer delegates to all"() { - setup: - def writers = new Writer[3] - Writer mockW1 = Mock() - Writer mockW2 = Mock() - writers[0] = mockW1 - // null in position 1 to check that we skip that - writers[2] = mockW2 - def writer = new MultiWriter(writers) - List trace = new LinkedList<>() - - when: - writer.start() - - then: - 1 * mockW1.start() - 1 * mockW2.start() - 0 * _ - - when: - writer.write(trace) - - then: - 1 * mockW1.write({ it == trace }) - 1 * mockW2.write({ it == trace }) - 0 * _ - - when: - def flushed = writer.flush() - - then: - 1 * mockW1.flush() >> true - 1 * mockW2.flush() >> true - 0 * _ - flushed - - when: - def notFlushed = writer.flush() - - then: - 1 * mockW1.flush() >> true - 1 * mockW2.flush() >> false - 0 * _ - !notFlushed - - when: - writer.close() - - then: - 1 * mockW1.close() - 1 * mockW2.close() - 0 * _ - - when: - writer.incrementDropCounts(0) - - then: - 1 * mockW1.incrementDropCounts(0) - 1 * mockW2.incrementDropCounts(0) - 0 * _ - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/PayloadDispatcherImplTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/PayloadDispatcherImplTest.groovy deleted file mode 100644 index 8aaa47062fa..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/PayloadDispatcherImplTest.groovy +++ /dev/null @@ -1,179 +0,0 @@ -package datadog.trace.common.writer - -import datadog.communication.ddagent.DDAgentFeaturesDiscovery -import datadog.metrics.api.statsd.StatsDClient -import datadog.metrics.impl.MonitoringImpl -import datadog.trace.api.DDSpanId -import datadog.trace.api.DDTraceId -import datadog.trace.api.datastreams.NoopPathwayContext -import datadog.trace.api.sampling.PrioritySampling -import datadog.trace.common.writer.ddagent.DDAgentApi -import datadog.trace.common.writer.ddagent.DDAgentMapperDiscovery -import datadog.trace.core.CoreTracer -import datadog.trace.core.DDSpan -import datadog.trace.core.DDSpanContext -import datadog.trace.core.PendingTrace -import datadog.trace.core.monitor.HealthMetrics -import datadog.trace.core.propagation.PropagationTags -import datadog.trace.test.util.DDSpecification -import java.nio.ByteBuffer -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicBoolean -import spock.lang.Shared -import spock.lang.Timeout - -class PayloadDispatcherImplTest extends DDSpecification { - - @Shared - MonitoringImpl monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS) - - @Timeout(10) - def "flush automatically when data limit is breached"() { - setup: - AtomicBoolean flushed = new AtomicBoolean() - HealthMetrics healthMetrics = Stub(HealthMetrics) - DDAgentFeaturesDiscovery discovery = Stub(DDAgentFeaturesDiscovery) - discovery.getTraceEndpoint() >> traceEndpoint - DDAgentApi api = Stub(DDAgentApi) - api.sendSerializedTraces(_) >> { - flushed.set(true) - return RemoteApi.Response.success(200) - } - PayloadDispatcherImpl dispatcher = new PayloadDispatcherImpl(new DDAgentMapperDiscovery(discovery), api, healthMetrics, monitoring) - List trace = [realSpan()] - when: - while (!flushed.get()) { - dispatcher.addTrace(trace) - } - - then: "the dispatcher has flushed" - flushed.get() - - where: - traceEndpoint << ["v0.5/traces", "v0.4/traces"] - } - - def "should flush buffer on demand"() { - setup: - HealthMetrics healthMetrics = Mock(HealthMetrics) - DDAgentFeaturesDiscovery discovery = Mock(DDAgentFeaturesDiscovery) - DDAgentApi api = Mock(DDAgentApi) - PayloadDispatcherImpl dispatcher = new PayloadDispatcherImpl(new DDAgentMapperDiscovery(discovery), api, healthMetrics, monitoring) - List trace = [realSpan()] - when: - for (int i = 0; i < traceCount; ++i) { - dispatcher.addTrace(trace) - } - dispatcher.flush() - then: - 2 * discovery.getTraceEndpoint() >> traceEndpoint - 1 * healthMetrics.onSerialize({ it > 0 }) - 1 * api.sendSerializedTraces({ it.traceCount() == traceCount }) >> RemoteApi.Response.success(200) - - where: - traceEndpoint | traceCount - "v0.4/traces" | 1 - "v0.4/traces" | 10 - "v0.4/traces" | 100 - "v0.5/traces" | 1 - "v0.5/traces" | 10 - "v0.5/traces" | 100 - } - - def "should report failed request to monitor"() { - setup: - HealthMetrics healthMetrics = Mock(HealthMetrics) - DDAgentFeaturesDiscovery discovery = Mock(DDAgentFeaturesDiscovery) - DDAgentApi api = Mock(DDAgentApi) - PayloadDispatcherImpl dispatcher = new PayloadDispatcherImpl(new DDAgentMapperDiscovery(discovery), api, healthMetrics, monitoring) - List trace = [realSpan()] - when: - for (int i = 0; i < traceCount; ++i) { - dispatcher.addTrace(trace) - } - dispatcher.flush() - then: - 2 * discovery.getTraceEndpoint() >> traceEndpoint - 1 * healthMetrics.onSerialize({ it > 0 }) - 1 * api.sendSerializedTraces({ it.traceCount() == traceCount }) >> RemoteApi.Response.failed(400) - - where: - traceEndpoint | traceCount - "v0.4/traces" | 1 - "v0.4/traces" | 10 - "v0.4/traces" | 100 - "v0.5/traces" | 1 - "v0.5/traces" | 10 - "v0.5/traces" | 100 - } - - def "should drop trace when there is no agent connectivity"() { - setup: - HealthMetrics healthMetrics = Mock(HealthMetrics) - DDAgentApi api = Mock(DDAgentApi) - DDAgentFeaturesDiscovery discovery = Mock(DDAgentFeaturesDiscovery) - PayloadDispatcherImpl dispatcher = new PayloadDispatcherImpl(new DDAgentMapperDiscovery(discovery), api, healthMetrics, monitoring) - List trace = [realSpan()] - discovery.getTraceEndpoint() >> null - when: - dispatcher.addTrace(trace) - then: - 1 * healthMetrics.onFailedPublish(PrioritySampling.UNSET,_) - } - - def "trace and span counts are reset after access"() { - setup: - HealthMetrics healthMetrics = Stub(HealthMetrics) - DDAgentApi api = Stub(DDAgentApi) - DDAgentFeaturesDiscovery discovery = Mock(DDAgentFeaturesDiscovery) { - it.getTraceEndpoint() >> "v0.4/traces" - } - PayloadDispatcherImpl dispatcher = new PayloadDispatcherImpl(new DDAgentMapperDiscovery(discovery), api, healthMetrics, monitoring) - - when: - dispatcher.addTrace([]) - dispatcher.onDroppedTrace(20) - dispatcher.onDroppedTrace(2) - Payload payload = dispatcher.newPayload(1, ByteBuffer.allocate(0)) - then: - payload.droppedSpans() == 22 - payload.droppedTraces() == 2 - when: - Payload newPayload = dispatcher.newPayload(1, ByteBuffer.allocate(0)) - then: - newPayload.droppedSpans() == 0 - newPayload.droppedTraces() == 0 - } - - - def realSpan() { - CoreTracer tracer = Stub(CoreTracer) - PendingTrace trace = Stub(PendingTrace) - trace.getTracer() >> tracer - trace.mapServiceName(_) >> { String serviceName -> serviceName } - def context = new DDSpanContext( - DDTraceId.ONE, - 1, - DDSpanId.ZERO, - null, - "", - "", - "", - PrioritySampling.UNSET, - "", - [:], - false, - "", - 0, - trace, - null, - null, - NoopPathwayContext.INSTANCE, - false, - PropagationTags.factory().empty()) - def span = new DDSpan("test", 0, context, null) - // Stub explicitly to avoid Spock fabricating a dummy DDSpan per call (slow reflection fallback). - trace.getRootSpan() >> span - return span - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/PrioritizationTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/PrioritizationTest.groovy deleted file mode 100644 index 8fa2f08eaa1..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/PrioritizationTest.groovy +++ /dev/null @@ -1,258 +0,0 @@ -package datadog.trace.common.writer - -import datadog.trace.common.writer.ddagent.FlushEvent -import datadog.trace.common.writer.ddagent.PrioritizationStrategy -import datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult -import datadog.trace.core.DDSpan -import datadog.trace.test.util.DDSpecification - -import java.util.concurrent.TimeUnit - -import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP -import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP -import static datadog.trace.api.sampling.PrioritySampling.UNSET -import static datadog.trace.api.sampling.PrioritySampling.USER_KEEP -import static datadog.trace.common.writer.ddagent.Prioritization.ENSURE_TRACE -import static datadog.trace.common.writer.ddagent.Prioritization.FAST_LANE -import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.* - -class PrioritizationTest extends DDSpecification { - - def "ensure trace strategy tries to send kept and unset priority traces to the primary queue until offer(..) is true, dropped traces to the secondary queue"() { - setup: - Queue primary = Mock(Queue) - Queue secondary = Mock(Queue) - PrioritizationStrategy blocking = ENSURE_TRACE.create(primary, secondary, null, { false }) - - when: - PublishResult publishResult = blocking.publish(Mock(DDSpan), priority, trace) - - then: - publishResult == ENQUEUED_FOR_SERIALIZATION - primaryOffers * primary.offer(trace) >> !primaryFull >> true - secondaryOffers * secondary.offer(trace) >> true - - where: - // spotless:off - trace | primaryFull | priority | primaryOffers | secondaryOffers - [] | true | UNSET | 2 | 0 - [] | true | SAMPLER_DROP | 0 | 1 - [] | true | SAMPLER_KEEP | 2 | 0 - [] | true | SAMPLER_DROP | 0 | 1 - [] | true | USER_KEEP | 2 | 0 - [] | false | UNSET | 1 | 0 - [] | false | SAMPLER_DROP | 0 | 1 - [] | false | SAMPLER_KEEP | 1 | 0 - [] | false | SAMPLER_DROP | 0 | 1 - [] | false | USER_KEEP | 1 | 0 - // spotless:on - } - - def "fast lane strategy sends kept and unset priority traces to the primary queue, dropped traces to the secondary queue"() { - setup: - Queue primary = Mock(Queue) - Queue secondary = Mock(Queue) - PrioritizationStrategy fastLane = FAST_LANE.create(primary, secondary, null, { false }) - - when: - PublishResult publishResult = fastLane.publish(Mock(DDSpan), priority, trace) - - then: - publishResult == DROPPED_BUFFER_OVERFLOW - primaryOffers * primary.offer(trace) - secondaryOffers * secondary.offer(trace) - - where: - // spotless:off - trace | priority | primaryOffers | secondaryOffers - [] | UNSET | 1 | 0 - [] | SAMPLER_DROP | 0 | 1 - [] | SAMPLER_KEEP | 1 | 0 - [] | SAMPLER_DROP | 0 | 1 - [] | USER_KEEP | 1 | 0 - // spotless:on - } - - def "FAST_LANE with active dropping policy sends kept and unset priority traces to the primary queue, drops all else"() { - setup: - Queue primary = Mock(Queue) - Queue secondary = Mock(Queue) - PrioritizationStrategy drop = FAST_LANE.create(primary, secondary, null, { true }) - - when: - PublishResult publishResult = drop.publish(Mock(DDSpan), priority, trace) - - then: - publishResult == expectedResult - primaryOffers * primary.offer(trace) >> true - 0 * secondary.offer(trace) - - where: - // spotless:off - trace | priority | primaryOffers | expectedResult - [] | UNSET | 1 | ENQUEUED_FOR_SERIALIZATION - [] | SAMPLER_DROP | 0 | DROPPED_BY_POLICY - [] | SAMPLER_KEEP | 1 | ENQUEUED_FOR_SERIALIZATION - [] | SAMPLER_DROP | 0 | DROPPED_BY_POLICY - [] | USER_KEEP | 1 | ENQUEUED_FOR_SERIALIZATION - // spotless:on - } - - def "#strategy strategy flushes primary queue"() { - setup: - Queue primary = Mock(Queue) - Queue secondary = Mock(Queue) - PrioritizationStrategy fastLane = strategy.create(primary, secondary, null, { false }) - when: - fastLane.flush(100, TimeUnit.MILLISECONDS) - then: - 1 * primary.offer({ it instanceof FlushEvent }) >> true - 0 * secondary.offer(_) - - where: - strategy << [FAST_LANE, ENSURE_TRACE] - } - - def "drop strategy respects force keep" () { - setup: - Queue primary = Mock(Queue) - PrioritizationStrategy drop = strategy.create(primary, null, null, { true }) - DDSpan root = Mock(DDSpan) - List trace = [root] - - when: - PublishResult publishResult = drop.publish(root, SAMPLER_DROP, trace) - - then: - publishResult == expectedResult - 1 * root.isForceKeep() >> forceKeep - (forceKeep ? 1 : 0) * primary.offer(trace) >> true - 0 * _ - - where: - strategy | forceKeep | expectedResult - FAST_LANE | true | ENQUEUED_FOR_SERIALIZATION - FAST_LANE | false | DROPPED_BY_POLICY - } - - def "ensure trace strategy tries to send kept and unset priority traces to the primary queue until offer(..) is true, dropped traces to the span sampling queue"() { - setup: - Queue primary = Mock(Queue) - Queue secondary = Mock(Queue) - Queue spanSampling = Mock(Queue) - PrioritizationStrategy blocking = ENSURE_TRACE.create(primary, secondary, spanSampling, { false }) - - when: - PublishResult publishResult = blocking.publish(Mock(DDSpan), priority, trace) - - then: - publishResult == expectedResult - primaryOffers * primary.offer(trace) >> !primaryFull >> true - 0 * secondary.offer(trace) // expect no traces sent to the secondary queue - singleSpanOffers * spanSampling.offer(trace) >> !singleSpanFull - - where: - trace | primaryFull | priority | primaryOffers | singleSpanOffers | singleSpanFull | expectedResult - [] | true | UNSET | 2 | 0 | false | ENQUEUED_FOR_SERIALIZATION - [] | true | SAMPLER_DROP | 0 | 1 | false | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING - [] | true | SAMPLER_KEEP | 2 | 0 | false | ENQUEUED_FOR_SERIALIZATION - [] | true | SAMPLER_DROP | 0 | 1 | false | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING - [] | true | USER_KEEP | 2 | 0 | false | ENQUEUED_FOR_SERIALIZATION - [] | false | UNSET | 1 | 0 | false | ENQUEUED_FOR_SERIALIZATION - [] | false | SAMPLER_DROP | 0 | 1 | false | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING - [] | false | SAMPLER_KEEP | 1 | 0 | false | ENQUEUED_FOR_SERIALIZATION - [] | false | SAMPLER_DROP | 0 | 1 | false | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING - [] | false | USER_KEEP | 1 | 0 | false | ENQUEUED_FOR_SERIALIZATION - [] | true | UNSET | 2 | 0 | true | ENQUEUED_FOR_SERIALIZATION - [] | true | SAMPLER_DROP | 0 | 1 | true | DROPPED_BUFFER_OVERFLOW - [] | true | SAMPLER_KEEP | 2 | 0 | true | ENQUEUED_FOR_SERIALIZATION - [] | true | SAMPLER_DROP | 0 | 1 | true | DROPPED_BUFFER_OVERFLOW - [] | true | USER_KEEP | 2 | 0 | true | ENQUEUED_FOR_SERIALIZATION - [] | false | UNSET | 1 | 0 | true | ENQUEUED_FOR_SERIALIZATION - [] | false | SAMPLER_DROP | 0 | 1 | true | DROPPED_BUFFER_OVERFLOW - [] | false | SAMPLER_KEEP | 1 | 0 | true | ENQUEUED_FOR_SERIALIZATION - [] | false | SAMPLER_DROP | 0 | 1 | true | DROPPED_BUFFER_OVERFLOW - [] | false | USER_KEEP | 1 | 0 | true | ENQUEUED_FOR_SERIALIZATION - } - - def "fast lane strategy sends kept and unset priority traces to the primary queue, dropped traces to the span sampling queue"() { - setup: - Queue primary = Mock(Queue) - Queue secondary = Mock(Queue) - Queue spanSampling = Mock(Queue) - PrioritizationStrategy fastLane = FAST_LANE.create(primary, secondary, spanSampling, { false }) - - when: - PublishResult publishResult = fastLane.publish(Mock(DDSpan), priority, trace) - - then: - publishResult == expectedResult - primaryOffers * primary.offer(trace) >> true - 0 * secondary.offer(trace) >> true // expect no traces sent to the secondary queue - singleSpanOffers * spanSampling.offer(trace) >> !singleSpanFull - - where: - trace | priority | primaryOffers | singleSpanOffers | singleSpanFull | expectedResult - [] | UNSET | 1 | 0 | false | ENQUEUED_FOR_SERIALIZATION - [] | SAMPLER_DROP | 0 | 1 | false | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING - [] | SAMPLER_KEEP | 1 | 0 | false | ENQUEUED_FOR_SERIALIZATION - [] | SAMPLER_DROP | 0 | 1 | false | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING - [] | USER_KEEP | 1 | 0 | false | ENQUEUED_FOR_SERIALIZATION - [] | UNSET | 1 | 0 | true | ENQUEUED_FOR_SERIALIZATION - [] | SAMPLER_DROP | 0 | 1 | true | DROPPED_BUFFER_OVERFLOW // span sampling queue is full - [] | SAMPLER_KEEP | 1 | 0 | true | ENQUEUED_FOR_SERIALIZATION - [] | SAMPLER_DROP | 0 | 1 | true | DROPPED_BUFFER_OVERFLOW // span sampling queue is full - [] | USER_KEEP | 1 | 0 | true | ENQUEUED_FOR_SERIALIZATION - } - - def "FAST_LANE with active dropping policy sends kept and unset priority traces to the primary queue, send to single span sampling all else"() { - setup: - Queue primary = Mock(Queue) - Queue secondary = Mock(Queue) - Queue spanSampling = Mock(Queue) - PrioritizationStrategy drop = FAST_LANE.create(primary, secondary, spanSampling, { true }) - - when: - PublishResult publishResult = drop.publish(Mock(DDSpan), priority, trace) - - then: - publishResult == expectedResult - primaryOffers * primary.offer(trace) >> true - 0 * secondary.offer(trace) - singleSpanOffers * spanSampling.offer(trace) >> true - - where: - trace | priority | primaryOffers | singleSpanOffers | expectedResult - [] | UNSET | 1 | 0 | ENQUEUED_FOR_SERIALIZATION - [] | SAMPLER_DROP | 0 | 1 | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING - [] | SAMPLER_KEEP | 1 | 0 | ENQUEUED_FOR_SERIALIZATION - [] | SAMPLER_DROP | 0 | 1 | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING - [] | USER_KEEP | 1 | 0 | ENQUEUED_FOR_SERIALIZATION - } - - def "span sampling drop strategy respects force keep" () { - setup: - Queue primary = Mock(Queue) - Queue spanSampling = Mock(Queue) - PrioritizationStrategy drop = strategy.create(primary, null, spanSampling, { true }) - DDSpan root = Mock(DDSpan) - List trace = [root] - - when: - PublishResult publishResult = drop.publish(root, SAMPLER_DROP, trace) - - then: - publishResult == expectedResult - 1 * root.isForceKeep() >> forceKeep - (forceKeep ? 1 : 0) * primary.offer(trace) >> true - (forceKeep ? 0 : 1) * spanSampling.offer(trace) >> !singleSpanFull - 0 * _ - - where: - strategy | forceKeep | singleSpanFull | expectedResult - FAST_LANE | true | true | ENQUEUED_FOR_SERIALIZATION - FAST_LANE | false | true | DROPPED_BUFFER_OVERFLOW - FAST_LANE | true | false | ENQUEUED_FOR_SERIALIZATION - FAST_LANE | false | false | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/SerializationTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/SerializationTest.groovy deleted file mode 100644 index ba273ca3e16..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/SerializationTest.groovy +++ /dev/null @@ -1,49 +0,0 @@ -package datadog.trace.common.writer - -import com.fasterxml.jackson.core.type.TypeReference -import com.fasterxml.jackson.databind.ObjectMapper -import datadog.trace.test.util.DDSpecification -import org.msgpack.core.MessagePack -import org.msgpack.jackson.dataformat.MessagePackFactory - -import static java.util.Collections.singletonMap - -class SerializationTest extends DDSpecification { - def "test json mapper serialization"() { - setup: - def mapper = new ObjectMapper() - def map = ["key1": "val1"] - def serializedMap = mapper.writeValueAsBytes(map) - def serializedList = "[${new String(serializedMap)}]".getBytes() - - when: - def result = mapper.readValue(serializedList, new TypeReference>>() {}) - - then: - result == [map] - new String(serializedList) == '[{"key1":"val1"}]' - } - - def "test msgpack mapper serialization"() { - setup: - def mapper = new ObjectMapper(new MessagePackFactory()) - // GStrings get odd results in the serializer. - def input = (1..1).collect { singletonMap("key$it".toString(), "val$it".toString()) } - def serializedMaps = input.collect { - mapper.writeValueAsBytes(it) - } - - def packer = MessagePack.newDefaultBufferPacker() - packer.packArrayHeader(serializedMaps.size()) - serializedMaps.each { - packer.writePayload(it) - } - def serializedList = packer.toByteArray() - - when: - def result = mapper.readValue(serializedList, new TypeReference>>() {}) - - then: - result == input - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/SpanSamplingWorkerTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/SpanSamplingWorkerTest.groovy deleted file mode 100644 index fcd16d08efd..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/SpanSamplingWorkerTest.groovy +++ /dev/null @@ -1,397 +0,0 @@ -package datadog.trace.common.writer - -import datadog.trace.api.sampling.PrioritySampling -import datadog.trace.common.sampling.SingleSpanSampler -import datadog.trace.core.DDSpan -import datadog.trace.core.monitor.HealthMetrics -import datadog.trace.test.util.DDSpecification -import static SpanSamplingWorker.DefaultSpanSamplingWorker - -import java.util.concurrent.CountDownLatch -import java.util.concurrent.LinkedBlockingDeque -import java.util.concurrent.TimeUnit - -class SpanSamplingWorkerTest extends DDSpecification { - - def "send only sampled spans to the sampled span queue"() { - setup: - Queue primaryQueue = new LinkedBlockingDeque<>(10) - Queue secondaryQueue = new LinkedBlockingDeque<>(10) - def droppingPolicy = { false } - SingleSpanSampler singleSpanSampler = Mock(SingleSpanSampler) - HealthMetrics healthMetrics = Mock(HealthMetrics) - SpanSamplingWorker worker = SpanSamplingWorker.build(10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, droppingPolicy) - worker.start() - DDSpan span1 = Mock(DDSpan) - DDSpan span2 = Mock(DDSpan) - DDSpan span3 = Mock(DDSpan) - singleSpanSampler.setSamplingPriority(span1) >> true - singleSpanSampler.setSamplingPriority(span2) >> false - singleSpanSampler.setSamplingPriority(span3) >> true - - when: - worker.getSpanSamplingQueue().offer([span1, span2, span3]) - - then: - primaryQueue.take() == [span1, span3] - secondaryQueue.take() == [span2] - - cleanup: - worker.close() - } - - def "handle multiple traces"() { - setup: - Queue primaryQueue = new LinkedBlockingDeque<>(10) - Queue secondaryQueue = new LinkedBlockingDeque<>(10) - def droppingPolicy = { false } - SingleSpanSampler singleSpanSampler = Mock(SingleSpanSampler) - HealthMetrics healthMetrics = Mock(HealthMetrics) - SpanSamplingWorker worker = SpanSamplingWorker.build(10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, droppingPolicy) - worker.start() - - DDSpan span1 = Mock(DDSpan) - DDSpan span2 = Mock(DDSpan) - DDSpan span3 = Mock(DDSpan) - singleSpanSampler.setSamplingPriority(span1) >> true - singleSpanSampler.setSamplingPriority(span2) >> false - singleSpanSampler.setSamplingPriority(span3) >> true - - DDSpan span4 = Mock(DDSpan) - DDSpan span5 = Mock(DDSpan) - singleSpanSampler.setSamplingPriority(span4) >> true - singleSpanSampler.setSamplingPriority(span5) >> false - - when: - worker.getSpanSamplingQueue().offer([span1, span2, span3]) - worker.getSpanSamplingQueue().offer([span4, span5]) - - then: - primaryQueue.take() == [span1, span3] - secondaryQueue.take() == [span2] - primaryQueue.take() == [span4] - secondaryQueue.take() == [span5] - - cleanup: - worker.close() - } - - def "skip traces with no sampled spans"() { - setup: - Queue primaryQueue = new LinkedBlockingDeque<>(10) - Queue secondaryQueue = new LinkedBlockingDeque<>(10) - def droppingPolicy = { false } - SingleSpanSampler singleSpanSampler = Mock(SingleSpanSampler) - HealthMetrics healthMetrics = Mock(HealthMetrics) - SpanSamplingWorker worker = SpanSamplingWorker.build(10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, droppingPolicy) - worker.start() - - DDSpan span1 = Mock(DDSpan) - DDSpan span2 = Mock(DDSpan) - DDSpan span3 = Mock(DDSpan) - singleSpanSampler.setSamplingPriority(span1) >> true - singleSpanSampler.setSamplingPriority(span2) >> false - singleSpanSampler.setSamplingPriority(span3) >> true - - DDSpan span4 = Mock(DDSpan) - DDSpan span5 = Mock(DDSpan) - singleSpanSampler.setSamplingPriority(span4) >> false - singleSpanSampler.setSamplingPriority(span5) >> false - - DDSpan span6 = Mock(DDSpan) - DDSpan span7 = Mock(DDSpan) - singleSpanSampler.setSamplingPriority(span6) >> true - singleSpanSampler.setSamplingPriority(span7) >> true - - when: - assert worker.getSpanSamplingQueue().offer([span1, span2, span3]) - assert worker.getSpanSamplingQueue().offer([span4, span5]) - assert worker.getSpanSamplingQueue().offer([span6, span7]) - - then: - primaryQueue.take() == [span1, span3] - secondaryQueue.take() == [span2] - secondaryQueue.take() == [span4, span5] - primaryQueue.take() == [span6, span7] - - cleanup: - worker.close() - } - - def "ignore empty traces"() { - setup: - Queue primaryQueue = new LinkedBlockingDeque<>(10) - Queue secondaryQueue = new LinkedBlockingDeque<>(10) - def droppingPolicy = { false } - SingleSpanSampler singleSpanSampler = Mock(SingleSpanSampler) - HealthMetrics healthMetrics = Mock(HealthMetrics) - SpanSamplingWorker worker = SpanSamplingWorker.build(10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, droppingPolicy) - worker.start() - - DDSpan span1 = Mock(DDSpan) - singleSpanSampler.setSamplingPriority(span1) >> true - - when: - assert worker.getSpanSamplingQueue().offer([]) - assert worker.getSpanSamplingQueue().offer([span1]) - - then: - primaryQueue.take() == [span1] - assert secondaryQueue.isEmpty() - - cleanup: - worker.close() - } - - def "update dropped traces metric when no tracer's spans have been sampled"() { - setup: - Queue primaryQueue = new LinkedBlockingDeque<>(10) - Queue secondaryQueue = new LinkedBlockingDeque<>(10) - def droppingPolicy = { false } - SingleSpanSampler singleSpanSampler = Mock(SingleSpanSampler) - HealthMetrics healthMetrics = Mock(HealthMetrics) - int expectedTraces = 1 - CountDownLatch latch = new CountDownLatch(expectedTraces) - SpanSamplingWorker worker = new DefaultSpanSamplingWorker(10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, droppingPolicy) { - @Override - protected void afterOnEvent() { - latch.countDown() - } - } - worker.start() - - DDSpan span1 = Mock(DDSpan) - DDSpan span2 = Mock(DDSpan) - span1.samplingPriority() >> PrioritySampling.USER_DROP - singleSpanSampler.setSamplingPriority(span1) >> false - singleSpanSampler.setSamplingPriority(span2) >> false - - - def queue = worker.getSpanSamplingQueue() - - when: - assert queue.offer([span1, span2]) - - then: - latch.await(10, TimeUnit.SECONDS) - - then: - assert primaryQueue.isEmpty() - secondaryQueue.take() == [span1, span2] - - then: - 1 * healthMetrics.onPublish([span1, span2], PrioritySampling.USER_DROP) - 0 * healthMetrics.onFailedPublish(_,_) - 0 * healthMetrics.onPartialPublish(_) - - cleanup: - worker.close() - } - - def "update dropped traces metric when primaryQueue is full"() { - setup: - Queue primaryQueue = new LinkedBlockingDeque<>(1) - Queue secondaryQueue = new LinkedBlockingDeque<>(10) - def droppingPolicy = { false } - primaryQueue.offer([]) // occupy the entire queue - SingleSpanSampler singleSpanSampler = Mock(SingleSpanSampler) - HealthMetrics healthMetrics = Mock(HealthMetrics) - int expectedTraces = 1 - CountDownLatch latch = new CountDownLatch(expectedTraces) - SpanSamplingWorker worker = new DefaultSpanSamplingWorker(10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, droppingPolicy) { - @Override - protected void afterOnEvent() { - latch.countDown() - } - } - worker.start() - - DDSpan span1 = Mock(DDSpan) - DDSpan span2 = Mock(DDSpan) - span1.samplingPriority() >> PrioritySampling.SAMPLER_DROP - singleSpanSampler.setSamplingPriority(span1) >> false - singleSpanSampler.setSamplingPriority(span2) >> true - - - def queue = worker.getSpanSamplingQueue() - - when: - assert queue.offer([span1, span2]) - - then: - latch.await(10, TimeUnit.SECONDS) - - then: - assert secondaryQueue.isEmpty() - - then: - 1 * healthMetrics.onFailedPublish(PrioritySampling.SAMPLER_DROP,_) - 0 * healthMetrics.onPublish(_, _) - 0 * healthMetrics.onPartialPublish(_) - - cleanup: - worker.close() - } - - def "update published traces metric when all trace's spans have been sampled"() { - setup: - Queue primaryQueue = new LinkedBlockingDeque<>(10) - Queue secondaryQueue = new LinkedBlockingDeque<>(10) - def droppingPolicy = { false } - SingleSpanSampler singleSpanSampler = Mock(SingleSpanSampler) - HealthMetrics healthMetrics = Mock(HealthMetrics) - SpanSamplingWorker worker = SpanSamplingWorker.build(10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, droppingPolicy) - worker.start() - - DDSpan span1 = Mock(DDSpan) - DDSpan span2 = Mock(DDSpan) - span1.samplingPriority() >> PrioritySampling.SAMPLER_DROP - singleSpanSampler.setSamplingPriority(span1) >> true - singleSpanSampler.setSamplingPriority(span2) >> true - - def queue = worker.getSpanSamplingQueue() - - when: - assert queue.offer([span1, span2]) - - then: - primaryQueue.take() == [span1, span2] - assert secondaryQueue.isEmpty() - - then: - 1 * healthMetrics.onPublish([span1, span2], PrioritySampling.SAMPLER_DROP) - 0 * healthMetrics.onFailedPublish(_,_) - 0 * healthMetrics.onPartialPublish(_) - - cleanup: - worker.close() - } - - def "update partial traces metric when some trace's spans have been dropped and sent to secondaryQueue"() { - setup: - Queue primaryQueue = new LinkedBlockingDeque<>(10) - Queue secondaryQueue = new LinkedBlockingDeque<>(10) - def droppingPolicy = { false } - SingleSpanSampler singleSpanSampler = Mock(SingleSpanSampler) - HealthMetrics healthMetrics = Mock(HealthMetrics) - SpanSamplingWorker worker = SpanSamplingWorker.build(10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, droppingPolicy) - worker.start() - - DDSpan span1 = Mock(DDSpan) - DDSpan span2 = Mock(DDSpan) - DDSpan span3 = Mock(DDSpan) - singleSpanSampler.setSamplingPriority(span1) >> false - singleSpanSampler.setSamplingPriority(span2) >> true - singleSpanSampler.setSamplingPriority(span3) >> false - - def queue = worker.getSpanSamplingQueue() - - when: - assert queue.offer([span1, span2, span3]) - - then: - primaryQueue.take() == [span2] - secondaryQueue.take() == [span1, span3] - - then: - 1 * healthMetrics.onPublish([span1, span2, span3], PrioritySampling.SAMPLER_DROP) - 0 * healthMetrics.onPartialPublish(_) - 0 * healthMetrics.onFailedPublish(_,_) - - cleanup: - worker.close() - } - - def "update partial traces metric when some trace's spans have been dropped and secondaryQueue is full or droppingPolicy is active"() { - setup: - Queue primaryQueue = new LinkedBlockingDeque<>(10) - Queue secondaryQueue = new LinkedBlockingDeque<>(secondaryQueueIsFull ? 1 : 10) - if (secondaryQueueIsFull) { - // occupy the entire queue - secondaryQueue.offer([]) - } - SingleSpanSampler singleSpanSampler = Mock(SingleSpanSampler) - HealthMetrics healthMetrics = Mock(HealthMetrics) - SpanSamplingWorker worker = new DefaultSpanSamplingWorker(10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, { droppingPolicy }) - worker.start() - - DDSpan span1 = Mock(DDSpan) - DDSpan span2 = Mock(DDSpan) - DDSpan span3 = Mock(DDSpan) - span1.samplingPriority() >> PrioritySampling.SAMPLER_DROP - singleSpanSampler.setSamplingPriority(span1) >> false - singleSpanSampler.setSamplingPriority(span2) >> true - singleSpanSampler.setSamplingPriority(span3) >> false - - def queue = worker.getSpanSamplingQueue() - - when: - assert queue.offer([span1, span2, span3]) - - then: - primaryQueue.take() == [span2] - - then: - 1 * healthMetrics.onPartialPublish(2) - 0 * healthMetrics.onFailedPublish(_,_) - 0 * healthMetrics.onPublish(_, _) - - cleanup: - worker.close() - - where: - droppingPolicy | secondaryQueueIsFull - true | false - false | true - true | true - } - - def "update FailedPublish metric when all trace's spans have been dropped and secondaryQueue is full or droppingPolicy is active"() { - setup: - Queue primaryQueue = new LinkedBlockingDeque<>(10) - Queue secondaryQueue = new LinkedBlockingDeque<>(secondaryQueueIsFull ? 1 : 10) - if (secondaryQueueIsFull) { - // occupy the entire queue - secondaryQueue.offer([]) - } - SingleSpanSampler singleSpanSampler = Mock(SingleSpanSampler) - HealthMetrics healthMetrics = Mock(HealthMetrics) - int expectedTraces = 1 - CountDownLatch latch = new CountDownLatch(expectedTraces) - SpanSamplingWorker worker = new DefaultSpanSamplingWorker(10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, { droppingPolicy }) { - @Override - protected void afterOnEvent() { - latch.countDown() - } - } - worker.start() - - DDSpan span1 = Mock(DDSpan) - DDSpan span2 = Mock(DDSpan) - span1.samplingPriority() >> PrioritySampling.SAMPLER_DROP - singleSpanSampler.setSamplingPriority(span1) >> false - singleSpanSampler.setSamplingPriority(span2) >> false - - def queue = worker.getSpanSamplingQueue() - - when: - assert queue.offer([span1, span2]) - - then: - latch.await(10, TimeUnit.SECONDS) - - then: - 1 * healthMetrics.onFailedPublish(PrioritySampling.SAMPLER_DROP,_) - 0 * healthMetrics.onPartialPublish(_) - 0 * healthMetrics.onPublish(_, _) - - cleanup: - worker.close() - - where: - droppingPolicy | secondaryQueueIsFull - true | false - false | true - true | true - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/TraceMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/TraceMapperTest.groovy deleted file mode 100644 index 6f2173e8374..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/TraceMapperTest.groovy +++ /dev/null @@ -1,101 +0,0 @@ -package datadog.trace.common.writer - -import datadog.trace.common.writer.ddagent.TraceMapper -import datadog.trace.common.writer.ddagent.TraceMapperV0_5 -import datadog.communication.serialization.ByteBufferConsumer -import datadog.communication.serialization.FlushingBuffer -import datadog.communication.serialization.msgpack.MsgPackWriter -import datadog.trace.core.DDSpan -import datadog.trace.core.test.DDCoreSpecification -import org.msgpack.core.MessagePack -import org.msgpack.core.MessageUnpacker - -import java.nio.ByteBuffer - -class TraceMapperTest extends DDCoreSpecification { - - def "test trace mapper v0.5"() { - setup: - def tracer = tracerBuilder().writer(new ListWriter()).build() - DDSpan span = (DDSpan) tracer.buildSpan("datadog", null).withTag("service.name", "my-service") - .withTag("elasticsearch.version", "7.0").start() - span.setBaggageItem("baggage", "item") - span.spanContext().setDataTop("mydata", "[1,2,3]") - def trace = [span] - - when: - TraceMapper traceMapper = new TraceMapperV0_5() - CapturingByteBufferConsumer sink = new CapturingByteBufferConsumer() - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1024, sink)) - packer.format(trace, traceMapper) - packer.flush() - - then: - sink.captured != null - ByteBuffer dictionaryBytes = traceMapper.dictionary.slice() - Map meta = new HashMap<>() - - MessageUnpacker dictionaryUnpacker = MessagePack.newDefaultUnpacker(dictionaryBytes) - int dictionaryLength = traceMapper.encoding.size() - String[] dictionary = new String[dictionaryLength] - for (int i = 0; i < dictionary.length; ++i) { - dictionary[i] = dictionaryUnpacker.unpackString() - } - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(sink.captured) - int traceCount = unpacker.unpackArrayHeader() - traceCount == 1 - for (int i = 0; i < traceCount; ++i) { - int arrayLength = unpacker.unpackArrayHeader() - arrayLength == 12 - String serviceName = dictionary[unpacker.unpackInt()] - serviceName == "my-service" - String operationName = dictionary[unpacker.unpackInt()] // operation name null - operationName == null - String resourceName = dictionary[unpacker.unpackInt()] - resourceName != null - long traceId = unpacker.unpackLong() - traceId == 1 - long spanId = unpacker.unpackLong() - spanId == 1 - long parentId = unpacker.unpackLong() - parentId == 0 - long start = unpacker.unpackLong() - start > 0 - long duration = unpacker.unpackLong() - duration >= 0 - int error = unpacker.unpackInt() - error == 0 - int metaHeader = unpacker.unpackMapHeader() - for (int j = 0; j < metaHeader; ++j) { - String key = dictionary[unpacker.unpackInt()] - key != null - String value = dictionary[unpacker.unpackInt()] - value != null - meta.put(key, value) - } - int metricsHeader = unpacker.unpackMapHeader() - for (int j = 0; j < metricsHeader; ++j) { - String key = dictionary[unpacker.unpackInt()] - key != null - unpacker.skipValue() - } - String type = dictionary[unpacker.unpackInt()] - type != null - - meta.findResult {it.getKey().contains('.mydata.') ? it.getValue() : null } == '[1,2,3]' - } - - cleanup: - tracer.close() - } - - static class CapturingByteBufferConsumer implements ByteBufferConsumer { - - ByteBuffer captured - - @Override - void accept(int messageCount, ByteBuffer buffer) { - captured = buffer - } - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/TraceProcessingWorkerTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/TraceProcessingWorkerTest.groovy deleted file mode 100644 index e6b076dcfa8..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/TraceProcessingWorkerTest.groovy +++ /dev/null @@ -1,430 +0,0 @@ -package datadog.trace.common.writer - -import datadog.trace.common.sampling.SingleSpanSampler -import datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult -import datadog.trace.core.CoreSpan -import datadog.trace.core.DDSpan -import datadog.trace.core.DDSpanContext -import datadog.trace.core.PendingTrace -import datadog.trace.core.monitor.HealthMetrics -import datadog.trace.bootstrap.instrumentation.api.SpanPostProcessor -import datadog.trace.test.util.DDSpecification -import spock.util.concurrent.PollingConditions - -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicInteger - -import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP -import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP -import static datadog.trace.api.sampling.PrioritySampling.UNSET -import static datadog.trace.api.sampling.PrioritySampling.USER_DROP -import static datadog.trace.api.sampling.PrioritySampling.USER_KEEP -import static datadog.trace.common.writer.ddagent.Prioritization.FAST_LANE -import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.ENQUEUED_FOR_SERIALIZATION - -class TraceProcessingWorkerTest extends DDSpecification { - - def conditions = new PollingConditions(timeout: 5, initialDelay: 0, factor: 1.25) - - def flushCountingPayloadDispatcher(AtomicInteger flushCounter) { - PayloadDispatcherImpl dispatcher = Mock(PayloadDispatcherImpl) - dispatcher.flush() >> { - flushCounter.incrementAndGet() - } - return dispatcher - } - - def "heartbeats should be triggered automatically when enabled"() { - setup: - AtomicInteger flushCount = new AtomicInteger() - TraceProcessingWorker worker = new TraceProcessingWorker(10, Stub(HealthMetrics), - flushCountingPayloadDispatcher(flushCount), { - false - }, - FAST_LANE, - 1, - TimeUnit.NANOSECONDS, - null - ) // stop heartbeats from being throttled - - when: "processor is started" - worker.start() - - then: "heartbeat occurs automatically" - conditions.eventually { - assert flushCount.get() > 0 - } - - cleanup: - worker.close() - } - - def "heartbeats should occur at least once per second when not throttled"() { - setup: - AtomicInteger flushCount = new AtomicInteger() - TraceProcessingWorker worker = new TraceProcessingWorker(10, Stub(HealthMetrics), - flushCountingPayloadDispatcher(flushCount), { - false - }, - FAST_LANE, - 1, - TimeUnit.NANOSECONDS, - null - ) // stop heartbeats from being throttled - def timeConditions = new PollingConditions(timeout: 1, initialDelay: 1, factor: 1.25) - - when: "processor is started" - worker.start() - - then: "heartbeat occurs automatically approximately once per second" - timeConditions.eventually { - assert flushCount.get() > 1 - } - - cleanup: - worker.close() - } - - def "a flush should clear the primary queue"() { - setup: - AtomicInteger flushCount = new AtomicInteger() - TraceProcessingWorker worker = new TraceProcessingWorker(10, Stub(HealthMetrics), - flushCountingPayloadDispatcher(flushCount), { - false - }, - FAST_LANE, - 100, TimeUnit.SECONDS, null) // prevent heartbeats from helping the flush happen - - when: "there is pending work it is completed before a flush" - // processing this span will throw an exception, but it should be caught - // and not disrupt the flush - worker.primaryQueue.offer([Mock(DDSpan)]) - worker.start() - boolean flushed = worker.flush(10, TimeUnit.SECONDS) - - then: "the flush succeeds, triggers a dispatch, and the queue is empty" - flushed - flushCount.get() == 1 - worker.primaryQueue.isEmpty() - - cleanup: - worker.close() - } - - def "should report failure if serialization fails"() { - setup: - Throwable theError = new IllegalStateException("thrown by test") - PayloadDispatcherImpl throwingDispatcher = Mock(PayloadDispatcherImpl) - throwingDispatcher.addTrace(_) >> { - throw theError - } - AtomicInteger errorReported = new AtomicInteger() - HealthMetrics healthMetrics = Mock(HealthMetrics) - healthMetrics.onFailedSerialize(_, theError) >> { - // do this manually with a counter, despite spock's - // lovely syntactical sugar so we don't have a race - // condition induced flaky test. All we care about - // is that an error was reported and that it was the - // right one - errorReported.incrementAndGet() - } - TraceProcessingWorker worker = new TraceProcessingWorker(10, healthMetrics, - throwingDispatcher, { - false - }, FAST_LANE, - 100, TimeUnit.SECONDS, null) // prevent heartbeats from helping the flush happen - worker.start() - - when: "a trace is processed but can't be passed on" - worker.publish(Mock(DDSpan), priority, [Mock(DDSpan)]) - - then: "the error is reported to the monitor" - conditions.eventually { - assert 1 == errorReported.get() - } - - cleanup: - worker.close() - - where: - priority << [SAMPLER_DROP, USER_DROP, SAMPLER_KEEP, USER_KEEP, UNSET] - } - - def "trace should be post-processed"() { - setup: - AtomicInteger acceptedCount = new AtomicInteger() - PayloadDispatcherImpl countingDispatcher = Mock(PayloadDispatcherImpl) - countingDispatcher.addTrace(_) >> { - acceptedCount.getAndIncrement() - } - HealthMetrics healthMetrics = Mock(HealthMetrics) - - def span1 = DDSpan.create("test", 0, Mock(DDSpanContext) { - getTraceCollector() >> Mock(PendingTrace) { - getCurrentTimeNano() >> 0 - } - }, []) - def processedSpan1 = false - - // Span 2 - should NOT be post-processed - def span2 = DDSpan.create("test", 0, Mock(DDSpanContext) { - getTraceCollector() >> Mock(PendingTrace) { - getCurrentTimeNano() >> 0 - } - }, []) - def processedSpan2 = false - - SpanPostProcessor.Holder.INSTANCE = Mock(SpanPostProcessor) { - process(span1, _) >> { processedSpan1 = true } - process(span2, _) >> { processedSpan2 = true } - } - - TraceProcessingWorker worker = new TraceProcessingWorker(10, healthMetrics, - countingDispatcher, { - false - }, FAST_LANE, 100, TimeUnit.SECONDS, null) - worker.start() - - when: "traces are submitted" - worker.publish(span1, SAMPLER_KEEP, [span1, span2]) - worker.publish(span2, SAMPLER_KEEP, [span1, span2]) - - then: "traces are passed through unless rejected on submission" - conditions.eventually { - assert processedSpan1 - assert processedSpan2 - } - - cleanup: - SpanPostProcessor.Holder.INSTANCE = SpanPostProcessor.Holder.NOOP - worker.close() - } - - def "traces should be processed"() { - setup: - AtomicInteger acceptedCount = new AtomicInteger() - PayloadDispatcherImpl countingDispatcher = Mock(PayloadDispatcherImpl) - countingDispatcher.addTrace(_) >> { - acceptedCount.getAndIncrement() - } - HealthMetrics healthMetrics = Mock(HealthMetrics) - TraceProcessingWorker worker = new TraceProcessingWorker(10, healthMetrics, - countingDispatcher, { - false - }, FAST_LANE, 100, TimeUnit.SECONDS, null) - // prevent heartbeats from helping the flush happen - worker.start() - - when: "traces are submitted" - int submitted = 0 - for (int i = 0; i < traceCount; ++i) { - PublishResult publishResult = worker.publish(Mock(DDSpan), priority, [Mock(DDSpan)]) - submitted += publishResult == ENQUEUED_FOR_SERIALIZATION ? 1 : 0 - } - - then: "traces are passed through unless rejected on submission" - 0 * healthMetrics.onFailedSerialize(_, _) - conditions.eventually { - assert submitted == acceptedCount.get() - } - - cleanup: - worker.close() - - where: - priority | traceCount | strategy - SAMPLER_DROP | 1 | FAST_LANE - USER_DROP | 1 | FAST_LANE - SAMPLER_KEEP | 1 | FAST_LANE - USER_KEEP | 1 | FAST_LANE - UNSET | 1 | FAST_LANE - SAMPLER_DROP | 10 | FAST_LANE - USER_DROP | 10 | FAST_LANE - SAMPLER_KEEP | 10 | FAST_LANE - USER_KEEP | 10 | FAST_LANE - UNSET | 10 | FAST_LANE - SAMPLER_DROP | 20 | FAST_LANE - USER_DROP | 20 | FAST_LANE - SAMPLER_KEEP | 20 | FAST_LANE - USER_KEEP | 20 | FAST_LANE - UNSET | 20 | FAST_LANE - SAMPLER_DROP | 100 | FAST_LANE - USER_DROP | 100 | FAST_LANE - SAMPLER_KEEP | 100 | FAST_LANE - USER_KEEP | 100 | FAST_LANE - UNSET | 100 | FAST_LANE - } - - def "flush of full queue after worker thread stopped will not flush but will return"() { - setup: - PayloadDispatcherImpl countingDispatcher = Mock(PayloadDispatcherImpl) - HealthMetrics healthMetrics = Mock(HealthMetrics) - TraceProcessingWorker worker = new TraceProcessingWorker(10, healthMetrics, - countingDispatcher, { - false - }, FAST_LANE, 100, TimeUnit.SECONDS, null) - worker.start() - worker.close() - int queueSize = 0 - while (worker.primaryQueue.offer([Mock(DDSpan)])) { - queueSize++ - } - - when: - boolean flushed = worker.flush(1, TimeUnit.SECONDS) - then: - !flushed - } - - def "send unsampled traces to the SpanProcessingWorker and expect only sampled spans dispatched when dropping policy is active"() { - setup: - HealthMetrics healthMetrics = Mock(HealthMetrics) - AtomicInteger acceptedCount = new AtomicInteger() - AtomicInteger acceptedSpanCount = new AtomicInteger() - PayloadDispatcherImpl countingDispatcher = Mock(PayloadDispatcherImpl) - countingDispatcher.addTrace(_) >> { - List traceList = it[0] - acceptedSpanCount.getAndAdd(traceList.size()) - acceptedCount.getAndIncrement() - } - AtomicInteger sampledSpansCount = new AtomicInteger() - SingleSpanSampler singleSpanSampler = new SingleSpanSampler() { - int counter = 0 - boolean setSamplingPriority(CoreSpan span) { - if (counter++ % 2 == 0) { - sampledSpansCount.incrementAndGet() - return true - } - // drop every other trace span - return false - } - } - TraceProcessingWorker worker = new TraceProcessingWorker(10, healthMetrics, countingDispatcher, { true }, FAST_LANE, 100, TimeUnit.SECONDS, singleSpanSampler) - worker.start() - - when: "traces are submitted" - for (int i = 0; i < traceCount; ++i) { - worker.publish(trace.get(0), priority, trace) - } - - then: "traces are passed through unless rejected on submission" - conditions.eventually { - assert acceptedTraces == acceptedCount.get() - assert acceptedSpans == acceptedSpanCount.get() - assert sampledSingleSpans == sampledSpansCount.get() - } - - cleanup: - worker.close() - - where: - priority | traceCount | acceptedTraces | acceptedSpans | sampledSingleSpans | trace - SAMPLER_DROP | 1 | 1 | 1 | 1 | [Mock(DDSpan)] - USER_DROP | 1 | 1 | 1 | 1 | [Mock(DDSpan), Mock(DDSpan)] - SAMPLER_DROP | 1 | 1 | 2 | 2 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_DROP | 1 | 1 | 2 | 2 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_DROP | 1 | 1 | 3 | 3 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_DROP | 2 | 1 | 1 | 1 | [Mock(DDSpan)] // expectedTraceCount = 1 b/o 2nd trace's only span gets unsampled - SAMPLER_DROP | 2 | 2 | 2 | 2 | [Mock(DDSpan), Mock(DDSpan)] - USER_DROP | 2 | 2 | 3 | 3 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_DROP | 2 | 2 | 4 | 4 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_DROP | 2 | 2 | 5 | 5 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_DROP | 10 | 5 | 5 | 5 | [Mock(DDSpan)] // expectedTraceCount = 5 b/o every 2nd trace's only span gets unsampled - USER_DROP | 10 | 10 | 10 | 10 | [Mock(DDSpan), Mock(DDSpan)] - SAMPLER_DROP | 10 | 10 | 15 | 15 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_DROP | 10 | 10 | 20 | 20 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_DROP | 10 | 10 | 25 | 25 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - // do not dispatch kept traces to the single span sampler - SAMPLER_KEEP | 1 | 1 | 1 | 0 | [Mock(DDSpan)] - USER_KEEP | 1 | 1 | 2 | 0 | [Mock(DDSpan), Mock(DDSpan)] - SAMPLER_KEEP | 1 | 1 | 3 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_KEEP | 1 | 1 | 4 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_KEEP | 1 | 1 | 5 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_KEEP | 2 | 2 | 2 | 0 | [Mock(DDSpan)] - SAMPLER_KEEP | 2 | 2 | 4 | 0 | [Mock(DDSpan), Mock(DDSpan)] - USER_KEEP | 2 | 2 | 6 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_KEEP | 2 | 2 | 8 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_KEEP | 2 | 2 | 10 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_KEEP | 10 | 10 | 10 | 0 | [Mock(DDSpan)] - USER_KEEP | 10 | 10 | 20 | 0 | [Mock(DDSpan), Mock(DDSpan)] - SAMPLER_KEEP | 10 | 10 | 30 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_KEEP | 10 | 10 | 40 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_KEEP | 10 | 10 | 50 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - } - - def "send unsampled traces to the SpanProcessingWorker and expect all spans dispatched when dropping policy is inactive"() { - setup: - HealthMetrics healthMetrics = Mock(HealthMetrics) - AtomicInteger chunksCount = new AtomicInteger() - AtomicInteger spansCount = new AtomicInteger() - PayloadDispatcherImpl countingDispatcher = Mock(PayloadDispatcherImpl) - countingDispatcher.addTrace(_) >> { - List traceList = it[0] - spansCount.getAndAdd(traceList.size()) - chunksCount.getAndIncrement() - } - AtomicInteger sampledSpansCount = new AtomicInteger() - SingleSpanSampler singleSpanSampler = new SingleSpanSampler() { - int counter = 0 - boolean setSamplingPriority(CoreSpan span) { - if (counter++ % 2 == 0) { - sampledSpansCount.incrementAndGet() - return true - } - // drop every other trace span - return false - } - } - TraceProcessingWorker worker = new TraceProcessingWorker(10, healthMetrics, countingDispatcher, { false }, FAST_LANE, 100, TimeUnit.SECONDS, singleSpanSampler) - worker.start() - - when: "traces are submitted" - for (int i = 0; i < traceCount; ++i) { - worker.publish(trace.get(0), priority, trace) - } - - then: "traces are passed through unless rejected on submission" - conditions.eventually { - assert expectedChunks == chunksCount.get() - assert expectedSpans == spansCount.get() - assert sampledSingleSpans == sampledSpansCount.get() - } - - cleanup: - worker.close() - - where: - priority | traceCount | expectedChunks | expectedSpans | sampledSingleSpans | trace - SAMPLER_DROP | 1 | 1 | 1 | 1 | [Mock(DDSpan)] - USER_DROP | 1 | 2 | 2 | 1 | [Mock(DDSpan), Mock(DDSpan)] - SAMPLER_DROP | 1 | 2 | 3 | 2 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_DROP | 1 | 2 | 4 | 2 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_DROP | 1 | 2 | 5 | 3 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_DROP | 2 | 2 | 2*1 | 1 | [Mock(DDSpan)] - SAMPLER_DROP | 2 | 2*2 | 2*2 | 2 | [Mock(DDSpan), Mock(DDSpan)] - USER_DROP | 2 | 2*2 | 2*3 | 3 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_DROP | 2 | 2*2 | 2*4 | 4 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_DROP | 2 | 2*2 | 2*5 | 5 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_DROP | 10 | 10 | 10 | 10/2*1 | [Mock(DDSpan)] - SAMPLER_DROP | 10 | 10*2 | 20 | 10/2*2 | [Mock(DDSpan), Mock(DDSpan)] - USER_DROP | 10 | 10*2 | 30 | 10/2*3 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_DROP | 10 | 10*2 | 40 | 10/2*4 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_DROP | 10 | 10*2 | 50 | 10/2*5 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - // do not dispatch kept traces to the single span sampler - SAMPLER_KEEP | 1 | 1 | 1 | 0 | [Mock(DDSpan)] - USER_KEEP | 1 | 1 | 2 | 0 | [Mock(DDSpan), Mock(DDSpan)] - SAMPLER_KEEP | 1 | 1 | 3 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_KEEP | 1 | 1 | 4 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_KEEP | 1 | 1 | 5 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_KEEP | 2 | 2 | 2*1 | 0 | [Mock(DDSpan)] - SAMPLER_KEEP | 2 | 2 | 2*2 | 0 | [Mock(DDSpan), Mock(DDSpan)] - USER_KEEP | 2 | 2 | 2*3 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_KEEP | 2 | 2 | 2*4 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_KEEP | 2 | 2 | 2*5 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_KEEP | 10 | 10 | 10 | 0 | [Mock(DDSpan)] - USER_KEEP | 10 | 10 | 20 | 0 | [Mock(DDSpan), Mock(DDSpan)] - SAMPLER_KEEP | 10 | 10 | 30 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - USER_KEEP | 10 | 10 | 40 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - SAMPLER_KEEP | 10 | 10 | 50 | 0 | [Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan), Mock(DDSpan)] - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/WriterFactoryTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/WriterFactoryTest.groovy deleted file mode 100644 index d20bd475cac..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/WriterFactoryTest.groovy +++ /dev/null @@ -1,242 +0,0 @@ -package datadog.trace.common.writer - -import static datadog.trace.api.config.TracerConfig.PRIORITIZATION_TYPE - -import datadog.communication.ddagent.DDAgentFeaturesDiscovery -import datadog.communication.ddagent.SharedCommunicationObjects -import datadog.trace.api.Config -import datadog.trace.api.config.OtlpConfig -import datadog.trace.api.intake.TrackType -import datadog.trace.common.sampling.Sampler -import datadog.trace.common.writer.ddagent.DDAgentApi -import datadog.trace.common.writer.ddagent.Prioritization -import datadog.trace.common.writer.ddintake.DDEvpProxyApi -import datadog.trace.common.writer.ddintake.DDIntakeApi -import datadog.trace.core.monitor.HealthMetrics -import datadog.trace.core.otlp.common.OtlpGrpcSender -import datadog.trace.core.otlp.common.OtlpHttpSender -import datadog.trace.test.util.DDSpecification -import groovy.json.JsonBuilder -import java.util.stream.Collectors -import okhttp3.Call -import okhttp3.HttpUrl -import okhttp3.MediaType -import okhttp3.OkHttpClient -import okhttp3.Protocol -import okhttp3.Request -import okhttp3.Response -import okhttp3.ResponseBody - -class WriterFactoryTest extends DDSpecification { - - def "test writer creation for #configuredType when agentHasEvpProxy=#hasEvpProxy evpProxySupportsCompression=#evpProxySupportsCompression ciVisibilityAgentless=#isCiVisibilityAgentlessEnabled"() { - setup: - def config = Mock(Config) - config.apiKey >> "my-api-key" - config.agentUrl >> "http://my-agent.url" - config.getEnumValue(PRIORITIZATION_TYPE, _, _) >> Prioritization.FAST_LANE - config.tracerMetricsEnabled >> true - config.isCiVisibilityEnabled() >> true - config.isCiVisibilityCodeCoverageEnabled() >> false - - // Mock agent info response - def response = buildHttpResponse(hasEvpProxy, evpProxySupportsCompression, HttpUrl.parse(config.agentUrl + "/info")) - - // Mock HTTP client that simulates delayed response for async feature discovery - def mockCall = Mock(Call) - def mockHttpClient = Mock(OkHttpClient) - mockCall.execute() >> { - // Add a delay - sleep(400) - return response - } - mockHttpClient.newCall(_ as Request) >> mockCall - - // Create SharedCommunicationObjects with mocked HTTP client - def sharedComm = new SharedCommunicationObjects() - sharedComm.agentHttpClient = mockHttpClient - sharedComm.agentUrl = HttpUrl.parse(config.agentUrl) - sharedComm.createRemaining(config) - - def sampler = Mock(Sampler) - - when: - config.ciVisibilityAgentlessEnabled >> isCiVisibilityAgentlessEnabled - - def writer = WriterFactory.createWriter(config, sharedComm, sampler, null, HealthMetrics.NO_OP, configuredType) - - def apis - def apiClasses - if (expectedApiClasses != null) { - apis = ((RemoteWriter) writer).apis - apiClasses = apis.stream().map(Object::getClass).collect(Collectors.toList()) - } else { - apis = Collections.emptyList() - apiClasses = Collections.emptyList() - } - - then: - writer.class == expectedWriterClass - expectedApiClasses == null || apiClasses == expectedApiClasses - expectedApiClasses == null || apis.stream().allMatch(api -> api.isCompressionEnabled() == isCompressionEnabled) - - where: - configuredType | hasEvpProxy | evpProxySupportsCompression | isCiVisibilityAgentlessEnabled | expectedWriterClass | expectedApiClasses | isCompressionEnabled - "LoggingWriter" | true | false | true | LoggingWriter | null | false - "PrintingWriter" | true | false | true | PrintingWriter | null | false - "TraceStructureWriter" | true | false | true | TraceStructureWriter | null | false - "MultiWriter:LoggingWriter,PrintingWriter" | true | false | true | MultiWriter | null | false - "DDIntakeWriter" | true | false | true | DDIntakeWriter | [DDIntakeApi] | true - "DDIntakeWriter" | true | false | false | DDIntakeWriter | [DDEvpProxyApi] | false - "DDIntakeWriter" | false | false | true | DDIntakeWriter | [DDIntakeApi] | true - "DDIntakeWriter" | false | false | false | DDIntakeWriter | [DDIntakeApi] | true - "DDAgentWriter" | true | false | true | DDIntakeWriter | [DDIntakeApi] | true - "DDAgentWriter" | true | false | false | DDIntakeWriter | [DDEvpProxyApi] | false - "DDAgentWriter" | true | true | false | DDIntakeWriter | [DDEvpProxyApi] | true - "DDAgentWriter" | false | false | true | DDIntakeWriter | [DDIntakeApi] | true - "DDAgentWriter" | false | false | false | DDAgentWriter | [DDAgentApi] | false - "not-found" | true | false | true | DDIntakeWriter | [DDIntakeApi] | true - "not-found" | true | false | false | DDIntakeWriter | [DDEvpProxyApi] | false - "not-found" | false | false | true | DDIntakeWriter | [DDIntakeApi] | true - "not-found" | false | false | false | DDAgentWriter | [DDAgentApi] | false - } - - def "test writer creation for #configuredType when agentHasEvpProxy=#hasEvpProxy llmObsAgentless=#isLlmObsAgentlessEnabled for LLM Observability"() { - setup: - def config = Mock(Config) - config.apiKey >> "my-api-key" - config.agentUrl >> "http://my-agent.url" - config.getEnumValue(PRIORITIZATION_TYPE, _, _) >> Prioritization.FAST_LANE - config.tracerMetricsEnabled >> true - config.isLlmObsEnabled() >> true - - // Mock agent info response - def response - if (agentRunning) { - response = buildHttpResponse(hasEvpProxy, true, HttpUrl.parse(config.agentUrl + "/info")) - } else { - response = buildHttpResponseNotOk(HttpUrl.parse(config.agentUrl + "/info")) - } - - // Mock HTTP client that simulates delayed response for async feature discovery - def mockCall = Mock(Call) - def mockHttpClient = Mock(OkHttpClient) - mockCall.execute() >> { - // Add a delay - sleep(400) - return response - } - mockHttpClient.newCall(_ as Request) >> mockCall - - // Create SharedCommunicationObjects with mocked HTTP client - def sharedComm = new SharedCommunicationObjects() - sharedComm.agentHttpClient = mockHttpClient - sharedComm.agentUrl = HttpUrl.parse(config.agentUrl) - sharedComm.createRemaining(config) - - def sampler = Mock(Sampler) - - when: - config.llmObsAgentlessEnabled >> isLlmObsAgentlessEnabled - - def writer = WriterFactory.createWriter(config, sharedComm, sampler, null, HealthMetrics.NO_OP, configuredType) - def llmObsApiClasses = ((RemoteWriter) writer).apis - .stream() - .filter(api -> { - try { - def trackTypeField = api.class.getDeclaredField("trackType") - trackTypeField.setAccessible(true) - return trackTypeField.get(api) == TrackType.LLMOBS - } catch (Exception e) { - return false - } - }) - .map(Object::getClass) - .collect(Collectors.toList()) - - then: - writer.class == expectedWriterClass - llmObsApiClasses == expectedLlmObsApiClasses - - where: - configuredType | agentRunning | hasEvpProxy | isLlmObsAgentlessEnabled |expectedWriterClass | expectedLlmObsApiClasses - "DDIntakeWriter" | true | true | false | DDIntakeWriter | [DDEvpProxyApi] - "DDIntakeWriter" | true | false | false | DDIntakeWriter | [DDIntakeApi] - "DDIntakeWriter" | false | false | false | DDIntakeWriter | [DDIntakeApi] - "DDIntakeWriter" | true | true | true | DDIntakeWriter | [DDIntakeApi] - "DDIntakeWriter" | true | false | true | DDIntakeWriter | [DDIntakeApi] - "DDIntakeWriter" | false | false | true | DDIntakeWriter | [DDIntakeApi] - } - - def "test writer creation for OtlpWriter wires #protocol+#compression"() { - setup: - def config = Mock(Config) - def headers = ["api-key": "secret"] - def readField = { Object instance, String fieldName -> - def field = instance.class.getDeclaredField(fieldName) - field.setAccessible(true) - return field.get(instance) - } - - config.getTraceFlushIntervalSeconds() >> 1.0f - config.getOtlpTracesEndpoint() >> endpoint - config.getOtlpTracesHeaders() >> headers - config.getOtlpTracesProtocol() >> protocol - config.getOtlpTracesCompression() >> compression - config.getOtlpTracesTimeout() >> 5000 - - when: - // OTLP branch in WriterFactory does not consult sharedComm or sampler, so nulls are safe here. - def writer = WriterFactory.createWriter(config, null, null, null, HealthMetrics.NO_OP, "OtlpWriter") - def sender = readField(writer, "sender") - - then: - writer.class == OtlpWriter - sender.class == expectedSenderClass - readField(sender, "url").toString() == expectedUrl - readField(sender, "headers") == headers - readField(sender, "gzip") == expectedGzip - - cleanup: - writer?.close() - - where: - protocol | compression | endpoint | expectedSenderClass | expectedUrl | expectedGzip - OtlpConfig.Protocol.HTTP_PROTOBUF | OtlpConfig.Compression.NONE | "http://otel-collector:4318/v1/traces" | OtlpHttpSender | "http://otel-collector:4318/v1/traces" | false - OtlpConfig.Protocol.HTTP_PROTOBUF | OtlpConfig.Compression.GZIP | "http://otel-collector:4318/v1/traces" | OtlpHttpSender | "http://otel-collector:4318/v1/traces" | true - OtlpConfig.Protocol.GRPC | OtlpConfig.Compression.NONE | "http://otel-collector:4317" | OtlpGrpcSender | "http://otel-collector:4317/opentelemetry.proto.collector.trace.v1.TraceService/Export" | false - } - - Response buildHttpResponse(boolean hasEvpProxy, boolean evpProxySupportsCompression, HttpUrl agentUrl) { - def endpoints = [] - if (hasEvpProxy && evpProxySupportsCompression) { - endpoints = [DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT] - } else if (hasEvpProxy) { - endpoints = [DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT] - } else { - endpoints = [DDAgentFeaturesDiscovery.V04_ENDPOINT] - } - - def response = [ - "version" : "7.40.0", - "endpoints" : endpoints, - ] - - def builder = new Response.Builder() - .code(200) - .message("OK") - .protocol(Protocol.HTTP_1_1) - .request(new Request.Builder().url(agentUrl.resolve("/info")).build()) - .body(ResponseBody.create(MediaType.parse("application/json"), new JsonBuilder(response).toString())) - return builder.build() - } - - Response buildHttpResponseNotOk(HttpUrl agentUrl) { - def builder = new Response.Builder() - .code(500) - .message("ERROR") - .protocol(Protocol.HTTP_1_1) - .request(new Request.Builder().url(agentUrl.resolve("/info")).build()) - return builder.build() - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy deleted file mode 100644 index 52ad90a9514..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy +++ /dev/null @@ -1,339 +0,0 @@ -package datadog.trace.common.writer.ddintake - -import com.fasterxml.jackson.core.type.TypeReference -import com.fasterxml.jackson.databind.ObjectMapper -import datadog.communication.EvpProxy -import datadog.communication.serialization.ByteBufferConsumer -import datadog.communication.serialization.FlushingBuffer -import datadog.communication.serialization.msgpack.MsgPackWriter -import datadog.trace.api.DDTags -import datadog.trace.api.civisibility.CiVisibilityWellKnownTags -import datadog.trace.api.intake.TrackType -import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes -import datadog.trace.bootstrap.instrumentation.api.ServiceNameSources -import datadog.trace.bootstrap.instrumentation.api.Tags -import datadog.trace.common.writer.Payload -import datadog.trace.core.DDSpan -import datadog.trace.core.test.DDCoreSpecification -import okhttp3.HttpUrl -import org.apache.commons.io.IOUtils -import org.msgpack.jackson.dataformat.MessagePackFactory -import spock.lang.Timeout - -import java.nio.ByteBuffer -import java.util.zip.GZIPInputStream - -import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT -import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT -import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer - -@Timeout(20) -class DDEvpProxyApiTest extends DDCoreSpecification { - - static CiVisibilityWellKnownTags wellKnownTags = new CiVisibilityWellKnownTags( - "my-runtime-id", "my-env", "my-language", - "my-runtime-name", "my-runtime-version", "my-runtime-vendor", - "my-os-arch", "my-os-platform", "my-os-version", "false") - - static String intakeSubdomain = "citestcycle-intake" - static msgPackMapper = new ObjectMapper(new MessagePackFactory()) - - static newAgentEvpProxy(String path) { - httpServer { - handlers { - post(path) { - if (request.contentType != "application/msgpack") { - response.status(400).send("wrong type: $request.contentType") - } else { - response.status(200).send() - } - } - } - } - } - - def "sending an empty list of traces returns no errors"() { - setup: - def path = buildAgentEvpProxyPath(evpProxyEndpoint, trackType, apiVersion) - def agentEvpProxy = newAgentEvpProxy(path) - def client = createEvpProxyApi(agentEvpProxy.address.toString(), evpProxyEndpoint, trackType, false) - def payload = prepareTraces(trackType, false, []) - - expect: - def clientResponse = client.sendSerializedTraces(payload) - clientResponse.success() - clientResponse.status().present - clientResponse.status().asInt == 200 - agentEvpProxy.getLastRequest().path == path - agentEvpProxy.getLastRequest().getHeader(EvpProxy.SUBDOMAIN_HEADER) == intakeSubdomain - - cleanup: - agentEvpProxy.close() - - where: - trackType | apiVersion | evpProxyEndpoint - TrackType.CITESTCYCLE | "v2" | V2_EVP_PROXY_ENDPOINT - } - - def "retries when backend returns 5xx"() { - setup: - def retry = 1 - def path = buildAgentEvpProxyPath(evpProxyEndpoint, trackType, apiVersion) - def agentEvpProxy = httpServer { - handlers { - post(path) { - if (retry < 5) { - response.status(503).send() - retry += 1 - } else { - response.status(200).send() - } - } - } - } - - def client = createEvpProxyApi(agentEvpProxy.address.toString(), evpProxyEndpoint, trackType, false) - def payload = prepareTraces(trackType, false, []) - - expect: - def clientResponse = client.sendSerializedTraces(payload) - clientResponse.success() - clientResponse.status().present - clientResponse.status().asInt == 200 - agentEvpProxy.getLastRequest().path == path - agentEvpProxy.getLastRequest().getHeader(EvpProxy.SUBDOMAIN_HEADER) == intakeSubdomain - - cleanup: - agentEvpProxy.close() - - where: - trackType | apiVersion | evpProxyEndpoint - TrackType.CITESTCYCLE | "v2" | V2_EVP_PROXY_ENDPOINT - } - - def "content is sent as MSGPACK"() { - setup: - def path = buildAgentEvpProxyPath(evpProxyEndpoint, trackType, apiVersion) - def agentEvpProxy = httpServer { - handlers { - post(path) { - response.send() - } - } - } - - def client = createEvpProxyApi(agentEvpProxy.address.toString(), evpProxyEndpoint, trackType, compressionEnabled) - def payload = prepareTraces(trackType, compressionEnabled, traces) - - expect: - client.sendSerializedTraces(payload).status() - agentEvpProxy.getLastRequest().contentType == "application/msgpack" - convertMap(agentEvpProxy.getLastRequest().body, compressionEnabled) == expectedRequestBody - - cleanup: - agentEvpProxy.close() - - where: - // spotless:off - trackType | apiVersion | evpProxyEndpoint | compressionEnabled | traces | expectedRequestBody - TrackType.CITESTCYCLE | "v2" | V2_EVP_PROXY_ENDPOINT | false | [] | [:] - - TrackType.CITESTCYCLE | "v2" | V2_EVP_PROXY_ENDPOINT | false | [[buildSpan(1L, "fakeType", ["service.name": "my-service"])]] | new TreeMap<>([ - "version" : 1, - "metadata": new TreeMap<>([ - "*": new TreeMap<>([ - "env" : "my-env", - "runtime-id" : "my-runtime-id", - "language" : "my-language", - (Tags.RUNTIME_NAME) : "my-runtime-name", - (Tags.RUNTIME_VERSION) : "my-runtime-version", - (Tags.RUNTIME_VENDOR) : "my-runtime-vendor", - (Tags.OS_ARCHITECTURE) : "my-os-arch", - (Tags.OS_PLATFORM) : "my-os-platform", - (Tags.OS_VERSION) : "my-os-version", - (DDTags.TEST_IS_USER_PROVIDED_SERVICE): "false" - ])]), - "events" : [new TreeMap<>([ - "type" : "span", - "version": 1, - "content": new TreeMap<>([ - "service" : "my-service", - "name" : "fakeOperation", - "resource" : "fakeResource", - "error" : 0, - "trace_id" : 1L, - "span_id" : 1L, - "parent_id": 0L, - "start" : 1000L, - "duration" : 10L, - "meta" : [(DDTags.DD_SVC_SRC): ServiceNameSources.MANUAL.toString()], - "metrics" : [:] - ]) - ])] - ]) - TrackType.CITESTCYCLE | "v2" | V2_EVP_PROXY_ENDPOINT | false | [[buildSpan(1L, InternalSpanTypes.TEST, ["test_suite_id": 123L, "test_module_id": 456L])]] | new TreeMap<>([ - "version" : 1, - "metadata": new TreeMap<>([ - "*": new TreeMap<>([ - "env" : "my-env", - "runtime-id" : "my-runtime-id", - "language" : "my-language", - (Tags.RUNTIME_NAME) : "my-runtime-name", - (Tags.RUNTIME_VERSION) : "my-runtime-version", - (Tags.RUNTIME_VENDOR) : "my-runtime-vendor", - (Tags.OS_ARCHITECTURE) : "my-os-arch", - (Tags.OS_PLATFORM) : "my-os-platform", - (Tags.OS_VERSION) : "my-os-version", - (DDTags.TEST_IS_USER_PROVIDED_SERVICE): "false" - ])]), - "events" : [new TreeMap<>([ - "type" : "test", - "version": 2, - "content": new TreeMap<>([ - "test_suite_id" : 123L, - "test_module_id": 456L, - "service" : "fakeService", - "name" : "fakeOperation", - "resource" : "fakeResource", - "error" : 0, - "trace_id" : 1L, - "span_id" : 1L, - "parent_id" : 0L, - "start" : 1000L, - "duration" : 10L, - "meta" : [:], - "metrics" : [:] - ]) - ])] - ]) - TrackType.CITESTCYCLE | "v2" | V2_EVP_PROXY_ENDPOINT | false | [[buildSpan(1L, InternalSpanTypes.TEST_SUITE_END, ["test_suite_id": 123L, "test_module_id": 456L])]] | new TreeMap<>([ - "version" : 1, - "metadata": new TreeMap<>([ - "*": new TreeMap<>([ - "env" : "my-env", - "runtime-id" : "my-runtime-id", - "language" : "my-language", - (Tags.RUNTIME_NAME) : "my-runtime-name", - (Tags.RUNTIME_VERSION) : "my-runtime-version", - (Tags.RUNTIME_VENDOR) : "my-runtime-vendor", - (Tags.OS_ARCHITECTURE) : "my-os-arch", - (Tags.OS_PLATFORM) : "my-os-platform", - (Tags.OS_VERSION) : "my-os-version", - (DDTags.TEST_IS_USER_PROVIDED_SERVICE): "false" - ])]), - "events" : [new TreeMap<>([ - "type" : "test_suite_end", - "version": 1, - "content": new TreeMap<>([ - "test_suite_id" : 123L, - "test_module_id": 456L, - "service" : "fakeService", - "name" : "fakeOperation", - "resource" : "fakeResource", - "error" : 0, - "start" : 1000L, - "duration" : 10L, - "meta" : [:], - "metrics" : [:] - ]) - ])] - ]) - TrackType.CITESTCYCLE | "v2" | V4_EVP_PROXY_ENDPOINT | true | [[buildSpan(1L, InternalSpanTypes.TEST_MODULE_END, ["test_module_id": 456L])]] | new TreeMap<>([ - "version" : 1, - "metadata": new TreeMap<>([ - "*": new TreeMap<>([ - "env" : "my-env", - "runtime-id" : "my-runtime-id", - "language" : "my-language", - (Tags.RUNTIME_NAME) : "my-runtime-name", - (Tags.RUNTIME_VERSION) : "my-runtime-version", - (Tags.RUNTIME_VENDOR) : "my-runtime-vendor", - (Tags.OS_ARCHITECTURE) : "my-os-arch", - (Tags.OS_PLATFORM) : "my-os-platform", - (Tags.OS_VERSION) : "my-os-version", - (DDTags.TEST_IS_USER_PROVIDED_SERVICE): "false" - ])]), - "events" : [new TreeMap<>([ - "type" : "test_module_end", - "version": 1, - "content": new TreeMap<>([ - "test_module_id": 456L, - "service" : "fakeService", - "name" : "fakeOperation", - "resource" : "fakeResource", - "error" : 0, - "start" : 1000L, - "duration" : 10L, - "meta" : [:], - "metrics" : [:] - ]) - ])] - ]) - - // spotless:on - ignore = traces.each { - it.each { - it.finish() - it.@durationNano = 10 - } - } - } - - static Map convertMap(byte[] bytes, boolean compressionEnabled) { - if (compressionEnabled) { - bytes = decompress(bytes) - } - return msgPackMapper.readValue(bytes, new TypeReference>() {}) - } - - static byte[] decompress(byte[] bytes) { - def baos = new ByteArrayOutputStream() - try (GZIPInputStream zip = new GZIPInputStream(new ByteArrayInputStream(bytes))) { - IOUtils.copy(zip, baos) - } - return baos.toByteArray() - } - - static class Traces implements ByteBufferConsumer { - int traceCount - ByteBuffer buffer - - @Override - void accept(int messageCount, ByteBuffer buffer) { - this.buffer = buffer - this.traceCount = messageCount - } - } - - def createEvpProxyApi(String agentUrl, String evpProxyEndpoint, TrackType trackType, boolean compressionEnabled) { - return DDEvpProxyApi.builder() - .agentUrl(HttpUrl.get(agentUrl)) - .evpProxyEndpoint(evpProxyEndpoint) - .trackType(trackType) - .compressionEnabled(compressionEnabled) - .build() - } - - def discoverMapper(TrackType trackType, boolean compressionEnabled) { - def mapperDiscover = new DDIntakeMapperDiscovery(trackType, wellKnownTags, compressionEnabled) - mapperDiscover.discover() - return mapperDiscover.getMapper() - } - - def buildAgentEvpProxyPath(String evpProxyEndpoint, TrackType trackType, String apiVersion) { - return "/" + evpProxyEndpoint + "api/" + apiVersion + "/" + trackType.name().toLowerCase() - } - - Payload prepareTraces(TrackType trackType, boolean compressionEnabled, List> traces) { - Traces traceCapture = new Traces() - def packer = new MsgPackWriter(new FlushingBuffer(1 << 20, traceCapture)) - def mapper = discoverMapper(trackType, compressionEnabled) - for (trace in traces) { - packer.format(trace, mapper) - } - packer.flush() - return mapper.newPayload() - .withBody(traceCapture.traceCount, - traces.isEmpty() ? ByteBuffer.allocate(0) : traceCapture.buffer) - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDIntakeApiTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDIntakeApiTest.groovy deleted file mode 100644 index ea3932854b4..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDIntakeApiTest.groovy +++ /dev/null @@ -1,360 +0,0 @@ -package datadog.trace.common.writer.ddintake - -import com.fasterxml.jackson.core.type.TypeReference -import com.fasterxml.jackson.databind.ObjectMapper -import datadog.communication.serialization.ByteBufferConsumer -import datadog.communication.serialization.FlushingBuffer -import datadog.communication.serialization.msgpack.MsgPackWriter -import datadog.trace.api.DDTags -import datadog.trace.api.civisibility.CiVisibilityWellKnownTags -import datadog.trace.api.intake.TrackType -import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes -import datadog.trace.bootstrap.instrumentation.api.ServiceNameSources -import datadog.trace.bootstrap.instrumentation.api.Tags -import datadog.trace.common.writer.Payload -import datadog.trace.core.DDSpan -import datadog.trace.core.test.DDCoreSpecification -import okhttp3.HttpUrl -import org.apache.commons.io.IOUtils -import org.msgpack.jackson.dataformat.MessagePackFactory -import spock.lang.Timeout - -import java.nio.ByteBuffer -import java.util.zip.GZIPInputStream - -import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer - -@Timeout(20) -class DDIntakeApiTest extends DDCoreSpecification { - - static CiVisibilityWellKnownTags wellKnownTags = new CiVisibilityWellKnownTags( - "my-runtime-id", "my-env", "my-language", - "my-runtime-name", "my-runtime-version", "my-runtime-vendor", - "my-os-arch", "my-os-platform", "my-os-version", "false") - - static String apiKey = "my-secret-apikey" - static msgPackMapper = new ObjectMapper(new MessagePackFactory()) - - static newIntake(String path) { - httpServer { - handlers { - post(path) { - if (request.contentType != "application/msgpack") { - response.status(400).send("wrong type: $request.contentType") - } else { - response.status(200).send() - } - } - } - } - } - - def "sending an empty list of traces returns no errors"() { - setup: - def path = buildIntakePath(trackType, apiVersion) - def intake = newIntake(path) - def client = createIntakeApi(intake.address.toString(), trackType) - def payload = prepareTraces(trackType, []) - - expect: - def clientResponse = client.sendSerializedTraces(payload) - clientResponse.success() - clientResponse.status().present - clientResponse.status().asInt == 200 - intake.getLastRequest().path == path - - cleanup: - intake.close() - - where: - trackType | apiVersion - TrackType.CITESTCYCLE | "v2" - } - - def "retries when backend returns 5xx"() { - setup: - def retry = 1 - def path = buildIntakePath(trackType, apiVersion) - def intake = httpServer { - handlers { - post(path) { - if (retry < 5) { - response.status(503).send() - retry += 1 - } else { - response.status(200).send() - } - } - } - } - - def client = createIntakeApi(intake.address.toString(), trackType) - def payload = prepareTraces(trackType, []) - - expect: - def clientResponse = client.sendSerializedTraces(payload) - clientResponse.success() - clientResponse.status().present - clientResponse.status().asInt == 200 - intake.getLastRequest().path == path - - cleanup: - intake.close() - - where: - trackType | apiVersion - TrackType.CITESTCYCLE | "v2" - } - - def "retries when backend returns 429 Too Many Requests"() { - setup: - def retry = 0 - def path = buildIntakePath(trackType, apiVersion) - def intake = httpServer { - handlers { - post(path) { - if (retry < 1) { - response.status(429).addHeader("x-ratelimit-reset", "0").send() - retry += 1 - } else { - response.status(200).send() - } - } - } - } - - def client = createIntakeApi(intake.address.toString(), trackType) - def payload = prepareTraces(trackType, []) - - expect: - def clientResponse = client.sendSerializedTraces(payload) - clientResponse.success() - clientResponse.status().present - clientResponse.status().asInt == 200 - intake.getLastRequest().path == path - - cleanup: - intake.close() - - where: - trackType | apiVersion - TrackType.CITESTCYCLE | "v2" - } - - def "content is sent as MSGPACK"() { - setup: - def path = buildIntakePath(trackType, apiVersion) - def intake = httpServer { - handlers { - post(path) { - response.send() - } - } - } - - def client = createIntakeApi(intake.address.toString(), trackType) - def payload = prepareTraces(trackType, traces) - - expect: - client.sendSerializedTraces(payload).status().present - intake.lastRequest.contentType == "application/msgpack" - convertMap(intake.lastRequest.body) == expectedRequestBody - - cleanup: - intake.close() - - where: - // spotless:off - trackType | apiVersion | traces | expectedRequestBody - TrackType.CITESTCYCLE | "v2" | [] | [:] - TrackType.CITESTCYCLE | "v2" | [[buildSpan(1L, "fakeType", ["service.name": "my-service"])]] | new TreeMap<>([ - "version" : 1, - "metadata": new TreeMap<>([ - "*": new TreeMap<>([ - "env" : "my-env", - "runtime-id" : "my-runtime-id", - "language" : "my-language", - (Tags.RUNTIME_NAME) : "my-runtime-name", - (Tags.RUNTIME_VERSION) : "my-runtime-version", - (Tags.RUNTIME_VENDOR) : "my-runtime-vendor", - (Tags.OS_ARCHITECTURE) : "my-os-arch", - (Tags.OS_PLATFORM) : "my-os-platform", - (Tags.OS_VERSION) : "my-os-version", - (DDTags.TEST_IS_USER_PROVIDED_SERVICE): "false", - ])]), - "events" : [new TreeMap<>([ - "type" : "span", - "version": 1, - "content": new TreeMap<>([ - "service" : "my-service", - "name" : "fakeOperation", - "resource" : "fakeResource", - "error" : 0, - "trace_id" : 1L, - "span_id" : 1L, - "parent_id": 0L, - "start" : 1000L, - "duration" : 10L, - "meta" : [(DDTags.DD_SVC_SRC):ServiceNameSources.MANUAL.toString()], - "metrics" : [:] - ]) - ])] - ]) - TrackType.CITESTCYCLE | "v2" | [[buildSpan(1L, InternalSpanTypes.TEST, ["test_suite_id": 123L, "test_module_id": 456L])]] | new TreeMap<>([ - "version" : 1, - "metadata": new TreeMap<>([ - "*": new TreeMap<>([ - "env" : "my-env", - "runtime-id" : "my-runtime-id", - "language" : "my-language", - (Tags.RUNTIME_NAME) : "my-runtime-name", - (Tags.RUNTIME_VERSION) : "my-runtime-version", - (Tags.RUNTIME_VENDOR) : "my-runtime-vendor", - (Tags.OS_ARCHITECTURE) : "my-os-arch", - (Tags.OS_PLATFORM) : "my-os-platform", - (Tags.OS_VERSION) : "my-os-version", - (DDTags.TEST_IS_USER_PROVIDED_SERVICE): "false" - ])]), - "events" : [new TreeMap<>([ - "type" : "test", - "version": 2, - "content": new TreeMap<>([ - "test_suite_id" : 123L, - "test_module_id": 456L, - "service" : "fakeService", - "name" : "fakeOperation", - "resource" : "fakeResource", - "error" : 0, - "trace_id" : 1L, - "span_id" : 1L, - "parent_id" : 0L, - "start" : 1000L, - "duration" : 10L, - "meta" : [:], - "metrics" : [:] - ]) - ])] - ]) - TrackType.CITESTCYCLE | "v2" | [[buildSpan(1L, InternalSpanTypes.TEST_SUITE_END, ["test_suite_id": 123L, "test_module_id": 456L])]] | new TreeMap<>([ - "version" : 1, - "metadata": new TreeMap<>([ - "*": new TreeMap<>([ - "env" : "my-env", - "runtime-id" : "my-runtime-id", - "language" : "my-language", - (Tags.RUNTIME_NAME) : "my-runtime-name", - (Tags.RUNTIME_VERSION) : "my-runtime-version", - (Tags.RUNTIME_VENDOR) : "my-runtime-vendor", - (Tags.OS_ARCHITECTURE) : "my-os-arch", - (Tags.OS_PLATFORM) : "my-os-platform", - (Tags.OS_VERSION) : "my-os-version", - (DDTags.TEST_IS_USER_PROVIDED_SERVICE): "false" - ])]), - "events" : [new TreeMap<>([ - "type" : "test_suite_end", - "version": 1, - "content": new TreeMap<>([ - "test_suite_id" : 123L, - "test_module_id": 456L, - "service" : "fakeService", - "name" : "fakeOperation", - "resource" : "fakeResource", - "error" : 0, - "start" : 1000L, - "duration" : 10L, - "meta" : [:], - "metrics" : [:] - ]) - ])] - ]) - TrackType.CITESTCYCLE | "v2" | [[buildSpan(1L, InternalSpanTypes.TEST_MODULE_END, ["test_module_id": 456L])]] | new TreeMap<>([ - "version" : 1, - "metadata": new TreeMap<>([ - "*": new TreeMap<>([ - "env" : "my-env", - "runtime-id" : "my-runtime-id", - "language" : "my-language", - (Tags.RUNTIME_NAME) : "my-runtime-name", - (Tags.RUNTIME_VERSION) : "my-runtime-version", - (Tags.RUNTIME_VENDOR) : "my-runtime-vendor", - (Tags.OS_ARCHITECTURE) : "my-os-arch", - (Tags.OS_PLATFORM) : "my-os-platform", - (Tags.OS_VERSION) : "my-os-version", - (DDTags.TEST_IS_USER_PROVIDED_SERVICE): "false" - ])]), - "events" : [new TreeMap<>([ - "type" : "test_module_end", - "version": 1, - "content": new TreeMap<>([ - "test_module_id": 456L, - "service" : "fakeService", - "name" : "fakeOperation", - "resource" : "fakeResource", - "error" : 0, - "start" : 1000L, - "duration" : 10L, - "meta" : [:], - "metrics" : [:] - ]) - ])] - ]) - // spotless:on - ignore = traces.each { - it.each { - it.finish() - it.@durationNano = 10 - } - } - } - - static Map convertMap(byte[] bytes) { - return msgPackMapper.readValue(decompress(bytes), new TypeReference>() {}) - } - - static byte[] decompress(byte[] bytes) { - def baos = new ByteArrayOutputStream() - try (GZIPInputStream zip = new GZIPInputStream(new ByteArrayInputStream(bytes))) { - IOUtils.copy(zip, baos) - } - return baos.toByteArray() - } - - static class Traces implements ByteBufferConsumer { - int traceCount - ByteBuffer buffer - - @Override - void accept(int messageCount, ByteBuffer buffer) { - this.buffer = buffer - this.traceCount = messageCount - } - } - - def createIntakeApi(String url, TrackType trackType) { - HttpUrl hostUrl = HttpUrl.get(url) - return DDIntakeApi.builder().hostUrl(hostUrl).trackType(trackType).apiKey(apiKey).build() - } - - def discoverMapper(TrackType trackType) { - def mapperDiscover = new DDIntakeMapperDiscovery(trackType, wellKnownTags, true) - mapperDiscover.discover() - return mapperDiscover.getMapper() - } - - def buildIntakePath(TrackType trackType, String apiVersion) { - return String.format("/api/%s/%s", apiVersion, trackType.name().toLowerCase()) - } - - Payload prepareTraces(TrackType trackType, List> traces) { - Traces traceCapture = new Traces() - def packer = new MsgPackWriter(new FlushingBuffer(1 << 20, traceCapture)) - def mapper = discoverMapper(trackType) - for (trace in traces) { - packer.format(trace, mapper) - } - packer.flush() - return mapper.newPayload() - .withBody(traceCapture.traceCount, - traces.isEmpty() ? ByteBuffer.allocate(0) : traceCapture.buffer) - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDIntakeTraceInterceptorTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDIntakeTraceInterceptorTest.groovy deleted file mode 100644 index 31c5c1c68ee..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDIntakeTraceInterceptorTest.groovy +++ /dev/null @@ -1,89 +0,0 @@ -package datadog.trace.common.writer.ddintake - -import datadog.trace.bootstrap.instrumentation.api.Tags -import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString -import datadog.trace.common.writer.ListWriter -import datadog.trace.core.test.DDCoreSpecification -import spock.lang.Timeout - -@Timeout(100) -class DDIntakeTraceInterceptorTest extends DDCoreSpecification { - - def writer = new ListWriter() - def tracer = tracerBuilder().writer(writer).build() - - def setup() { - tracer.addTraceInterceptor(DDIntakeTraceInterceptor.INSTANCE) - } - - def cleanup() { - tracer?.close() - } - - def "test normalization for dd intake"() { - setup: - tracer.buildSpan("datadog", "my-operation-name") - .withResourceName("my-resource-name") - .withSpanType("my-span-type") - .withServiceName("my-service-name") - .withTag("some-tag-key", "some-tag-value") - .withTag("env", " My_____Env ") - .withTag(Tags.HTTP_STATUS, httpStatus) - .start().finish() - writer.waitForTraces(1) - - expect: - def trace = writer.firstTrace() - trace.size() == 1 - - def span = trace[0] - - span.getServiceName() == "my-service-name" - span.getOperationName() == "my_operation_name" - span.getResourceName() == "my-resource-name" - span.getSpanType() == "my-span-type" - span.getTag("some-tag-key") == "some-tag-value" - span.getTag("env") == "my_env" - span.getTag(Tags.HTTP_STATUS) == expectedHttpStatus - - where: - httpStatus | expectedHttpStatus - null | null - "" | null - "500" | 500 - 500 | 500 - 600 | null - } - - def "test normalization does not implicitly convert span type"() { - setup: - def originalSpanType = UTF8BytesString.create("a UTF8 span type") - tracer.buildSpan("datadog", "my-operation-name") - .withSpanType(originalSpanType) - .start().finish() - - when: - writer.waitForTraces(1) - - then: - def trace = writer.firstTrace() - trace.size() == 1 - - def span = trace[0] - span.type == originalSpanType - } - - def "test default env setting"() { - setup: - tracer.buildSpan("datadog", "my-operation-name").start().finish() - writer.waitForTraces(1) - - expect: - def trace = writer.firstTrace() - trace.size() == 1 - - def span = trace[0] - - span.getTag("env") == "none" - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDIntakeTrackTypeResolverTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDIntakeTrackTypeResolverTest.groovy deleted file mode 100644 index d13bff4e1d9..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDIntakeTrackTypeResolverTest.groovy +++ /dev/null @@ -1,24 +0,0 @@ -package datadog.trace.common.writer.ddintake - -import datadog.trace.api.Config -import datadog.trace.api.intake.TrackType -import datadog.trace.test.util.DDSpecification - -class DDIntakeTrackTypeResolverTest extends DDSpecification { - - def "should return the correct TrackType"() { - setup: - Config config = Mock(Config) - config.ciVisibilityEnabled >> ciVisibilityEnabled - config.ciVisibilityAgentlessEnabled >> ciVisibilityAgentlessEnabled - - expect: - DDIntakeTrackTypeResolver.resolve(config) == expectedTrackType - - where: - ciVisibilityEnabled | ciVisibilityAgentlessEnabled | expectedTrackType - false | false | TrackType.NOOP - true | false | TrackType.CITESTCYCLE - true | true | TrackType.CITESTCYCLE - } -} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java new file mode 100644 index 00000000000..1452c640509 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java @@ -0,0 +1,742 @@ +package datadog.trace.common.writer; + +import static datadog.trace.api.ProtocolVersion.V0_5; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.communication.http.OkHttpUtils; +import datadog.communication.serialization.ByteBufferConsumer; +import datadog.communication.serialization.FlushingBuffer; +import datadog.communication.serialization.msgpack.MsgPackWriter; +import datadog.metrics.api.Monitoring; +import datadog.metrics.api.statsd.StatsDClient; +import datadog.metrics.impl.MonitoringImpl; +import datadog.trace.agent.test.server.http.JavaTestHttpServer; +import datadog.trace.api.Config; +import datadog.trace.api.ProcessTags; +import datadog.trace.api.ProtocolVersion; +import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags; +import datadog.trace.common.sampling.RateByServiceTraceSampler; +import datadog.trace.common.writer.RemoteApi.Response; +import datadog.trace.common.writer.ddagent.DDAgentApi; +import datadog.trace.common.writer.ddagent.TraceMapper; +import datadog.trace.common.writer.ddagent.TraceMapperV0_4; +import datadog.trace.common.writer.ddagent.TraceMapperV0_5; +import datadog.trace.common.writer.ddagent.TraceMapperV1; +import datadog.trace.core.DDCoreJavaSpecification; +import datadog.trace.core.DDSpan; +import datadog.trace.core.DDSpanContext; +import datadog.trace.core.propagation.PropagationTags; +import datadog.trace.junit.utils.tabletest.TableTestTypeConverters; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.msgpack.jackson.dataformat.MessagePackFactory; +import org.tabletest.junit.TableTest; +import org.tabletest.junit.TypeConverterSources; + +@Timeout(20) +@TypeConverterSources(TableTestTypeConverters.class) +public class DDAgentApiTest extends DDCoreJavaSpecification { + + static final Monitoring monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS); + static final ObjectMapper mapper = new ObjectMapper(new MessagePackFactory()); + + // --- Helper: create a minimal agent server responding 200 to PUT latestVersion --- + + static JavaTestHttpServer newAgent(String latestVersion) { + return JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.put( + latestVersion, + api -> { + if (!"application/msgpack".equals(api.getRequest().getContentType())) { + api.getResponse() + .status(400) + .send("wrong type: " + api.getRequest().getContentType()); + } else if (api.getRequest().getContentLength() <= 0) { + api.getResponse().status(400).send("no content"); + } else { + api.getResponse().status(200).send(); + } + }))); + } + + // --- Tests --- + + @TableTest({ + "scenario | agentVersion | protocolVersion", + "v0.3 traces | 'v0.3/traces' | V0_4 ", + "v0.4 traces | 'v0.4/traces' | V0_4 ", + "v0.5 traces | 'v0.5/traces' | V0_5 ", + "v1.0 traces | 'v1.0/traces' | V1_0 " + }) + void testSendingAnEmptyListOfTracesReturnsNoErrors( + String agentVersion, ProtocolVersion protocolVersion) { + JavaTestHttpServer agent = newAgent(agentVersion); + try { + DDAgentApi client = createAgentApi(agent.getAddress().toString(), protocolVersion).api; + Payload payload = prepareTraces(agentVersion, emptyList()); + Response response = client.sendSerializedTraces(payload); + assertTrue(response.success()); + assertTrue(response.status().isPresent()); + assertEquals(200, response.status().getAsInt()); + assertEquals("/" + agentVersion, agent.getLastRequest().getPath()); + } finally { + agent.close(); + } + } + + @Test + void testResponseBodyPropagatedInCaseOfNon200Response() { + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.put( + "v0.4/traces", + api -> api.getResponse().status(400).send("Test error")))); + try { + DDAgentApi client = createAgentApi(agent.getAddress().toString()).api; + Payload payload = prepareTraces("v0.4/traces", emptyList()); + Response clientResponse = client.sendSerializedTraces(payload); + assertFalse(clientResponse.success()); + assertTrue(clientResponse.status().isPresent()); + assertEquals(400, clientResponse.status().getAsInt()); + assertEquals("Test error", clientResponse.response()); + } finally { + agent.close(); + } + } + + @Test + void testNon200Response() { + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> { + h.put("v0.4/traces", api -> api.getResponse().status(404).send()); + h.put("v0.3/traces", api -> api.getResponse().status(404).send()); + })); + try { + DDAgentApi client = createAgentApi(agent.getAddress().toString()).api; + Payload payload = prepareTraces("v0.3/traces", emptyList()); + Response clientResponse = client.sendSerializedTraces(payload); + assertFalse(clientResponse.success()); + assertTrue(clientResponse.status().isPresent()); + assertEquals(404, clientResponse.status().getAsInt()); + assertEquals("/v0.3/traces", agent.getLastRequest().getPath()); + } finally { + agent.close(); + } + } + + @Test + void testContentIsSentAsMsgpackEmptyTraces() throws IOException { + String agentVersion = "v0.3/traces"; + List> traces = emptyList(); + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + s -> s.handlers(h -> h.put(agentVersion, api -> api.getResponse().send()))); + try { + DDAgentApi client = createAgentApi(agent.getAddress().toString()).api; + Payload payload = prepareTraces(agentVersion, traces); + assertTrue(client.sendSerializedTraces(payload).success()); + assertEquals("application/msgpack", agent.getLastRequest().getContentType()); + assertEquals( + "true", agent.getLastRequest().getHeaders().get("Datadog-Client-Computed-Top-Level")); + assertEquals("java", agent.getLastRequest().getHeaders().get("Datadog-Meta-Lang")); + assertEquals( + System.getProperty("java.version", "unknown"), + agent.getLastRequest().getHeaders().get("Datadog-Meta-Lang-Version")); + assertEquals( + "Stubbed-Test-Version", + agent.getLastRequest().getHeaders().get("Datadog-Meta-Tracer-Version")); + assertEquals( + String.valueOf(traces.size()), + agent.getLastRequest().getHeaders().get("X-Datadog-Trace-Count")); + assertEquals( + String.valueOf(payload.droppedTraces()), + agent.getLastRequest().getHeaders().get("Datadog-Client-Dropped-P0-Traces")); + assertEquals( + String.valueOf(payload.droppedSpans()), + agent.getLastRequest().getHeaders().get("Datadog-Client-Dropped-P0-Spans")); + assertEquals(emptyList(), convertList(agentVersion, agent.getLastRequest().getBody())); + } finally { + agent.close(); + } + } + + @Test + void testContentIsSentAsMsgpackServiceSpan() throws IOException { + String agentVersion = "v0.4/traces"; + DDSpan span = + buildSpan( + 1L, + "service.name", + "my-service", + PropagationTags.factory() + .fromHeaderValue(PropagationTags.HeaderType.DATADOG, "_dd.p.usr=123")); + span.finish(); + setDurationNano(span, 10L); + List> traces = singletonList(singletonList(span)); + + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + s -> s.handlers(h -> h.put(agentVersion, api -> api.getResponse().send()))); + try { + DDAgentApi client = createAgentApi(agent.getAddress().toString()).api; + Payload payload = prepareTraces(agentVersion, traces); + assertTrue(client.sendSerializedTraces(payload).success()); + assertEquals("application/msgpack", agent.getLastRequest().getContentType()); + assertEquals( + "true", agent.getLastRequest().getHeaders().get("Datadog-Client-Computed-Top-Level")); + assertEquals("java", agent.getLastRequest().getHeaders().get("Datadog-Meta-Lang")); + assertEquals( + System.getProperty("java.version", "unknown"), + agent.getLastRequest().getHeaders().get("Datadog-Meta-Lang-Version")); + assertEquals( + "Stubbed-Test-Version", + agent.getLastRequest().getHeaders().get("Datadog-Meta-Tracer-Version")); + assertEquals( + String.valueOf(traces.size()), + agent.getLastRequest().getHeaders().get("X-Datadog-Trace-Count")); + assertEquals( + String.valueOf(payload.droppedTraces()), + agent.getLastRequest().getHeaders().get("Datadog-Client-Dropped-P0-Traces")); + assertEquals( + String.valueOf(payload.droppedSpans()), + agent.getLastRequest().getHeaders().get("Datadog-Client-Dropped-P0-Spans")); + + Map meta = new TreeMap<>(); + meta.put("thread.name", Thread.currentThread().getName()); + meta.put("_dd.p.usr", "123"); + meta.put("_dd.p.dm", "-1"); + meta.put("_dd.p.ksr", "1"); + meta.put("_dd.svc_src", "m"); + if (Config.get().isExperimentalPropagateProcessTagsEnabled() + && ProcessTags.getTagsForSerialization() != null) { + meta.put("_dd.tags.process", ProcessTags.getTagsForSerialization().toString()); + } + Map metrics = new TreeMap<>(); + metrics.put(DDSpanContext.PRIORITY_SAMPLING_KEY, 1); + metrics.put(InstrumentationTags.DD_TOP_LEVEL.toString(), 1); + metrics.put(RateByServiceTraceSampler.SAMPLING_AGENT_RATE, 1.0); + metrics.put("thread.id", Thread.currentThread().getId()); + Map spanMap = new TreeMap<>(); + spanMap.put("duration", 10); + spanMap.put("error", 0); + spanMap.put("meta", meta); + spanMap.put("metrics", metrics); + spanMap.put("name", "fakeOperation"); + spanMap.put("parent_id", 0); + spanMap.put("resource", "fakeResource"); + spanMap.put("service", "my-service"); + spanMap.put("span_id", 1); + spanMap.put("start", 1000); + spanMap.put("trace_id", 1); + spanMap.put("type", "fakeType"); + List>> expectedRequestBody = singletonList(singletonList(spanMap)); + assertDeepEquals( + expectedRequestBody, convertList(agentVersion, agent.getLastRequest().getBody())); + } finally { + agent.close(); + } + } + + @Test + void testContentIsSentAsMsgpackResourceSpan() throws IOException { + String agentVersion = "v0.4/traces"; + DDSpan span = + buildSpan( + 100L, + "resource.name", + "my-resource", + PropagationTags.factory() + .fromHeaderValue(PropagationTags.HeaderType.DATADOG, "_dd.p.usr=123")); + span.finish(); + setDurationNano(span, 10L); + List> traces = singletonList(singletonList(span)); + + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + s -> s.handlers(h -> h.put(agentVersion, api -> api.getResponse().send()))); + try { + DDAgentApi client = createAgentApi(agent.getAddress().toString()).api; + Payload payload = prepareTraces(agentVersion, traces); + assertTrue(client.sendSerializedTraces(payload).success()); + assertEquals("application/msgpack", agent.getLastRequest().getContentType()); + assertEquals( + "true", agent.getLastRequest().getHeaders().get("Datadog-Client-Computed-Top-Level")); + assertEquals("java", agent.getLastRequest().getHeaders().get("Datadog-Meta-Lang")); + assertEquals( + System.getProperty("java.version", "unknown"), + agent.getLastRequest().getHeaders().get("Datadog-Meta-Lang-Version")); + assertEquals( + "Stubbed-Test-Version", + agent.getLastRequest().getHeaders().get("Datadog-Meta-Tracer-Version")); + assertEquals( + String.valueOf(traces.size()), + agent.getLastRequest().getHeaders().get("X-Datadog-Trace-Count")); + assertEquals( + String.valueOf(payload.droppedTraces()), + agent.getLastRequest().getHeaders().get("Datadog-Client-Dropped-P0-Traces")); + assertEquals( + String.valueOf(payload.droppedSpans()), + agent.getLastRequest().getHeaders().get("Datadog-Client-Dropped-P0-Spans")); + + Map meta = new TreeMap<>(); + meta.put("thread.name", Thread.currentThread().getName()); + meta.put("_dd.p.usr", "123"); + meta.put("_dd.p.dm", "-1"); + meta.put("_dd.p.ksr", "1"); + if (Config.get().isExperimentalPropagateProcessTagsEnabled() + && ProcessTags.getTagsForSerialization() != null) { + meta.put("_dd.tags.process", ProcessTags.getTagsForSerialization().toString()); + } + Map metrics = new TreeMap<>(); + metrics.put(DDSpanContext.PRIORITY_SAMPLING_KEY, 1); + metrics.put(InstrumentationTags.DD_TOP_LEVEL.toString(), 1); + metrics.put(RateByServiceTraceSampler.SAMPLING_AGENT_RATE, 1.0); + metrics.put("thread.id", Thread.currentThread().getId()); + Map spanMap = new TreeMap<>(); + spanMap.put("duration", 10); + spanMap.put("error", 0); + spanMap.put("meta", meta); + spanMap.put("metrics", metrics); + spanMap.put("name", "fakeOperation"); + spanMap.put("parent_id", 0); + spanMap.put("resource", "my-resource"); + spanMap.put("service", "fakeService"); + spanMap.put("span_id", 1); + spanMap.put("start", 100000); + spanMap.put("trace_id", 1); + spanMap.put("type", "fakeType"); + List>> expectedRequestBody = singletonList(singletonList(spanMap)); + assertDeepEquals( + expectedRequestBody, convertList(agentVersion, agent.getLastRequest().getBody())); + } finally { + agent.close(); + } + } + + @TableTest({ + "scenario | agentVersion ", + "v0.3 traces | 'v0.3/traces'", + "v0.4 traces | 'v0.4/traces'", + "v0.5 traces | 'v0.5/traces'" + }) + void testApiResponseListenersSee200Responses(String agentVersion) { + AtomicReference>> agentResponse = new AtomicReference<>(null); + RemoteResponseListener responseListener = + (endpoint, responseJson) -> agentResponse.set(responseJson); + + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.put( + agentVersion, + api -> { + int status = api.getRequest().getContentLength() > 0 ? 200 : 500; + api.getResponse().status(status).send("{\"hello\":{}}"); + }))); + try { + DDAgentApi client = createAgentApi(agent.getAddress().toString()).api; + client.addResponseListener(responseListener); + List> traces = Arrays.asList(emptyList(), emptyList(), emptyList()); + Payload payload = prepareTraces(agentVersion, traces); + payload.withDroppedTraces(1); + payload.withDroppedTraces(3); + + client.sendSerializedTraces(payload); + + Map> response = agentResponse.get(); + assertTrue(response != null && response.containsKey("hello")); + assertEquals("java", agent.getLastRequest().getHeaders().get("Datadog-Meta-Lang")); + assertEquals( + System.getProperty("java.version", "unknown"), + agent.getLastRequest().getHeaders().get("Datadog-Meta-Lang-Version")); + assertEquals( + "Stubbed-Test-Version", + agent.getLastRequest().getHeaders().get("Datadog-Meta-Tracer-Version")); + assertEquals("3", agent.getLastRequest().getHeaders().get("X-Datadog-Trace-Count")); + assertEquals( + String.valueOf(payload.droppedTraces()), + agent.getLastRequest().getHeaders().get("Datadog-Client-Dropped-P0-Traces")); + assertEquals( + String.valueOf(payload.droppedSpans()), + agent.getLastRequest().getHeaders().get("Datadog-Client-Dropped-P0-Spans")); + } finally { + agent.close(); + } + } + + @Test + void testApiDowngradesToV3IfV04NotAvailable() { + JavaTestHttpServer v3Agent = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.put( + "v0.3/traces", + api -> { + int status = api.getRequest().getContentLength() > 0 ? 200 : 500; + api.getResponse().status(status).send(); + }))); + try { + DDAgentApi client = createAgentApi(v3Agent.getAddress().toString()).api; + Payload payload = prepareTraces("v0.4/traces", emptyList()); + assertTrue(client.sendSerializedTraces(payload).success()); + assertEquals("/v0.3/traces", v3Agent.getLastRequest().getPath()); + } finally { + v3Agent.close(); + } + } + + @TableTest({ + "scenario | endpointVersion | delayTrace | badPort", + "v0.4 ok | 'v0.4' | 0 | false ", + "v0.3 bad port | 'v0.3' | 0 | true ", + "v0.4 short delay | 'v0.4' | 500 | false ", + "v0.3 long delay | 'v0.3' | 30000 | false " + }) + void testApiDowngradesToV3IfTimeoutExceeded( + String endpointVersion, int delayTrace, boolean badPort) { + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> { + h.put( + "v0.3/traces", + api -> { + int status = api.getRequest().getContentLength() > 0 ? 200 : 500; + api.getResponse().status(status).send(); + }); + h.put( + "v0.4/traces", + api -> { + if (delayTrace > 0) { + try { + Thread.sleep(delayTrace); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + int status = api.getRequest().getContentLength() > 0 ? 200 : 500; + api.getResponse().status(status).send(); + }); + })); + try { + int port = badPort ? 999 : agent.getAddress().getPort(); + String url = "http://" + agent.getAddress().getHost() + ":" + port; + DDAgentApi client = createAgentApi(url).api; + Payload payload = prepareTraces("v0.4/traces", emptyList()); + Response result = client.sendSerializedTraces(payload); + assertEquals(!badPort, result.success()); + if (!badPort) { + assertEquals("/" + endpointVersion + "/traces", agent.getLastRequest().getPath()); + } + } finally { + agent.close(); + } + } + + // all the tested traces are empty and it just so happens that + // arrays and maps take the same amount of space in messagepack, so + // all the sizes match, except in v0.5 where there is 1 byte for a + // 2 element array header and 1 byte for an empty dictionary + @TableTest({ + "scenario | agentVersion | expectedLength | traceCount", + "v0.4 empty | 'v0.4/traces' | 1 | 0 ", + "v0.4 2 traces | 'v0.4/traces' | 3 | 2 ", + "v0.4 15 traces | 'v0.4/traces' | 16 | 15 ", + "v0.4 16 traces | 'v0.4/traces' | 19 | 16 ", + "v0.4 65535 traces | 'v0.4/traces' | 65538 | 65535 ", + "v0.4 65536 traces | 'v0.4/traces' | 65541 | 65536 ", + "v0.5 empty | 'v0.5/traces' | 3 | 0 ", + "v0.5 2 traces | 'v0.5/traces' | 5 | 2 ", + "v0.5 15 traces | 'v0.5/traces' | 18 | 15 ", + "v0.5 16 traces | 'v0.5/traces' | 21 | 16 ", + "v0.5 65535 traces | 'v0.5/traces' | 65540 | 65535 ", + "v0.5 65536 traces | 'v0.5/traces' | 65543 | 65536 " + }) + void testVerifyContentLength(String agentVersion, long expectedLength, int traceCount) { + List> traces = generateEmptyTraces(traceCount); + AtomicLong receivedContentLength = new AtomicLong(); + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.put( + agentVersion, + api -> { + receivedContentLength.set(api.getRequest().getContentLength()); + api.getResponse().status(200).send(); + }))); + try { + DDAgentApi client = createAgentApi(agent.getAddress().toString()).api; + Payload payload = prepareTraces(agentVersion, traces); + assertTrue(client.sendSerializedTraces(payload).success()); + assertEquals(expectedLength, receivedContentLength.get()); + } finally { + agent.close(); + } + } + + @Test + void testEmbeddedHttpClientRejectsAsyncRequests() throws Exception { + JavaTestHttpServer agent = newAgent("v0.5/traces"); + try { + AgentApiPair pair = createAgentApi(agent.getAddress().toString()); + pair.discovery.discover(); + Field httpClientField = DDAgentApi.class.getDeclaredField("httpClient"); + httpClientField.setAccessible(true); + OkHttpClient httpClient = (OkHttpClient) httpClientField.get(pair.api); + ExecutorService httpExecutorService = httpClient.dispatcher().executorService(); + assertThrows(RejectedExecutionException.class, () -> httpExecutorService.execute(() -> {})); + assertTrue(httpExecutorService.isShutdown()); + } finally { + agent.close(); + } + } + + @Test + void testMetaStructSupportOnTheEncodedSpans() throws IOException { + String agentVersion = "v0.4/traces"; + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + s -> s.handlers(h -> h.put(agentVersion, api -> api.getResponse().send()))); + try { + DDAgentApi client = createAgentApi(agent.getAddress().toString()).api; + DDSpan span = buildSpan(1L, "fakeType", Collections.emptyMap()); + span.setMetaStruct("meta_1", "Hello World!"); + Map meta2 = new HashMap<>(); + meta2.put("Hello", " World!"); + span.setMetaStruct("meta_2", meta2); + Payload payload = prepareTraces(agentVersion, singletonList(singletonList(span))); + assertTrue(client.sendSerializedTraces(payload).success()); + + List>> body = + convertList(agentVersion, agent.getLastRequest().getBody()); + @SuppressWarnings("unchecked") + Map metaStruct = (Map) body.get(0).get(0).get("meta_struct"); + assertEquals(2, metaStruct.size()); + assertEquals("Hello World!", mapper.readValue(metaStruct.get("meta_1"), String.class)); + @SuppressWarnings("unchecked") + Map actualMeta2 = mapper.readValue(metaStruct.get("meta_2"), Map.class); + assertEquals(meta2, actualMeta2); + } finally { + agent.close(); + } + } + + // --- Inner types --- + + static class AgentApiPair { + final DDAgentFeaturesDiscovery discovery; + final DDAgentApi api; + + AgentApiPair(DDAgentFeaturesDiscovery discovery, DDAgentApi api) { + this.discovery = discovery; + this.api = api; + } + } + + static class TracesCapture implements ByteBufferConsumer { + int traceCount; + ByteBuffer buffer; + + @Override + public void accept(int messageCount, ByteBuffer buffer) { + this.buffer = buffer; + this.traceCount = messageCount; + } + } + + // --- Helper methods --- + + AgentApiPair createAgentApi(String url, ProtocolVersion protocolVersion) { + HttpUrl agentUrl = HttpUrl.get(url); + OkHttpClient client = OkHttpUtils.buildHttpClient(agentUrl, 1000); + DDAgentFeaturesDiscovery discovery = + new DDAgentFeaturesDiscovery(client, monitoring, agentUrl, protocolVersion, true, false); + return new AgentApiPair( + discovery, new DDAgentApi(client, agentUrl, discovery, monitoring, false)); + } + + AgentApiPair createAgentApi(String url) { + return createAgentApi(url, V0_5); + } + + Payload prepareTraces(String agentVersion, List> traces) { + TracesCapture traceCapture = new TracesCapture(); + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1 << 20, traceCapture)); + + TraceMapper traceMapper; + if ("v1.0/traces".equals(agentVersion)) { + traceMapper = new TraceMapperV1(); + } else if ("v0.5/traces".equals(agentVersion)) { + traceMapper = new TraceMapperV0_5(); + } else { + traceMapper = new TraceMapperV0_4(); + } + + for (List trace : traces) { + packer.format(trace, traceMapper); + } + packer.flush(); + + return traceMapper + .newPayload() + .withBody( + traceCapture.traceCount, + traces.isEmpty() ? ByteBuffer.allocate(0) : traceCapture.buffer); + } + + static List>> convertList(String agentVersion, byte[] bytes) + throws IOException { + if ("v0.5/traces".equals(agentVersion)) { + return convertListV5(bytes); + } + List>> returnVal = + mapper.readValue(bytes, new TypeReference>>>() {}); + for (List> trace : returnVal) { + for (TreeMap span : trace) { + @SuppressWarnings("unchecked") + Map meta = (Map) span.get("meta"); + if (meta != null) { + meta.remove("runtime-id"); + meta.remove("language"); + } + } + } + return returnVal; + } + + static List>> convertListV5(byte[] bytes) throws IOException { + List>> traces = + mapper.readValue(bytes, new TypeReference>>>() {}); + List>> maps = new ArrayList<>(traces.size()); + for (List> trace : traces) { + List> mapTrace = new ArrayList<>(); + for (List span : trace) { + TreeMap map = new TreeMap<>(); + if (!span.isEmpty()) { + map.put("service", span.get(0)); + map.put("name", span.get(1)); + map.put("resource", span.get(2)); + map.put("trace_id", span.get(3)); + map.put("span_id", span.get(4)); + map.put("parent_id", span.get(5)); + map.put("start", span.get(6)); + map.put("duration", span.get(7)); + map.put("error", span.get(8)); + map.put("meta", span.get(9)); + map.put("metrics", span.get(10)); + map.put("type", span.get(11)); + + @SuppressWarnings("unchecked") + Map meta = (Map) map.get("meta"); + if (meta != null) { + meta.remove("runtime-id"); + meta.remove("language"); + } + } + mapTrace.add(map); + } + maps.add(mapTrace); + } + return maps; + } + + static List> generateEmptyTraces(int count) { + List> traces = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + traces.add(emptyList()); + } + return traces; + } + + static void setDurationNano(DDSpan span, long duration) { + try { + Field field = DDSpan.class.getDeclaredField("durationNano"); + field.setAccessible(true); + field.setLong(span, duration); + } catch (NoSuchFieldException | IllegalAccessException e) { + Assertions.fail("Could not set durationNano: " + e.getMessage()); + } + } + + @SuppressWarnings("unchecked") + static void assertDeepEquals(Object expected, Object actual) { + if (expected == null && actual == null) { + return; + } + if (expected == null || actual == null) { + Assertions.fail("Expected " + expected + " but got " + actual); + } + if (expected instanceof Map) { + assertInstanceOf(Map.class, actual, "Expected Map but got " + actual.getClass()); + Map expectedMap = (Map) expected; + Map actualMap = (Map) actual; + assertEquals(expectedMap.size(), actualMap.size(), "Map size mismatch"); + for (Map.Entry entry : expectedMap.entrySet()) { + assertTrue(actualMap.containsKey(entry.getKey()), "Missing key: " + entry.getKey()); + assertDeepEquals(entry.getValue(), actualMap.get(entry.getKey())); + } + } else if (expected instanceof List) { + assertInstanceOf(List.class, actual, "Expected List but got " + actual.getClass()); + List expectedList = (List) expected; + List actualList = (List) actual; + assertEquals(expectedList.size(), actualList.size(), "List size mismatch"); + for (int i = 0; i < expectedList.size(); i++) { + assertDeepEquals(expectedList.get(i), actualList.get(i)); + } + } else if (expected instanceof Number && actual instanceof Number) { + if (expected instanceof Float + || expected instanceof Double + || actual instanceof Float + || actual instanceof Double) { + assertEquals(((Number) expected).doubleValue(), ((Number) actual).doubleValue(), 0.0001); + } else { + assertEquals(((Number) expected).longValue(), ((Number) actual).longValue()); + } + } else { + assertEquals(expected, actual); + } + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterCombinedTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterCombinedTest.java new file mode 100644 index 00000000000..284d40b1736 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterCombinedTest.java @@ -0,0 +1,837 @@ +package datadog.trace.common.writer; + +import static datadog.trace.api.ProtocolVersion.V0_5; +import static datadog.trace.api.config.GeneralConfig.EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED; +import static datadog.trace.common.writer.ddagent.Prioritization.ENSURE_TRACE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.communication.http.OkHttpUtils; +import datadog.communication.serialization.FlushingBuffer; +import datadog.communication.serialization.msgpack.MsgPackWriter; +import datadog.metrics.api.statsd.StatsDClient; +import datadog.metrics.impl.MonitoringImpl; +import datadog.trace.agent.test.server.http.JavaTestHttpServer; +import datadog.trace.api.Config; +import datadog.trace.api.ProcessTags; +import datadog.trace.common.writer.ddagent.DDAgentApi; +import datadog.trace.common.writer.ddagent.TraceMapper; +import datadog.trace.common.writer.ddagent.TraceMapperV0_4; +import datadog.trace.common.writer.ddagent.TraceMapperV0_5; +import datadog.trace.core.CoreTracer; +import datadog.trace.core.DDCoreJavaSpecification; +import datadog.trace.core.DDSpan; +import datadog.trace.core.monitor.HealthMetrics; +import datadog.trace.core.monitor.TracerHealthMetrics; +import datadog.trace.junit.utils.config.WithConfig; +import datadog.trace.test.util.Flaky; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Phaser; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import okhttp3.HttpUrl; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.mockito.Mockito; +import org.tabletest.junit.TableTest; + +@Timeout(value = 10, unit = TimeUnit.SECONDS) +class DDAgentWriterCombinedTest extends DDCoreJavaSpecification { + + // DDAgentWriter default buffer size (matches private DDAgentWriter.BUFFER_SIZE) + private static final int AGENT_WRITER_BUFFER_SIZE = 1024; + + MonitoringImpl monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS); + Phaser phaser = new Phaser(); + + // Only used to create spans + CoreTracer dummyTracer; + + @BeforeEach + void setup() { + // Register for two threads. + phaser.register(); + phaser.register(); + dummyTracer = tracerBuilder().writer(new ListWriter()).build(); + } + + @AfterEach + void cleanup() { + if (dummyTracer != null) { + dummyTracer.close(); + } + } + + @BeforeEach + void resetProcessTags() { + // Sync ProcessTags.enabled with the current Config (which may be modified by @WithConfig) + ProcessTags.reset(Config.get()); + } + + List createMinimalTrace() { + // Use buildSpan from DDCoreJavaSpecification to create a real DDSpan with minimal fields + DDSpan span = buildSpan(0L, "", Collections.emptyMap()); + return Collections.singletonList(span); + } + + @Test + void noInteractionsBecauseOfInitialFlush() { + DDAgentApi api = Mockito.mock(DDAgentApi.class); + DDAgentWriter writer = + DDAgentWriter.builder() + .agentApi(api) + .traceBufferSize(8) + .monitoring(monitoring) + .flushIntervalMilliseconds(-1) + .build(); + writer.start(); + + writer.flush(); + + // then: 0 * _ (no interactions at all on mocked api) + verifyNoMoreInteractions(api); + + writer.close(); + } + + @TableTest({ + "scenario | agentVersion ", + "v0.3 | 'v0.3/traces'", + "v0.4 | 'v0.4/traces'", + "v0.5 | 'v0.5/traces'" + }) + void testHappyPath(String agentVersion) { + DDAgentApi api = mock(DDAgentApi.class); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + DDAgentWriter writer = + DDAgentWriter.builder() + .featureDiscovery(discovery) + .agentApi(api) + .traceBufferSize(1024) + .monitoring(monitoring) + .flushIntervalMilliseconds(-1) + .build(); + writer.start(); + DDSpan span = (DDSpan) dummyTracer.buildSpan("datadog", "fakeOperation").start(); + List trace = Collections.singletonList(span); + + when(discovery.getTraceEndpoint()).thenReturn(agentVersion); + when(api.sendSerializedTraces(argThat(payload -> payload.traceCount() == 2))) + .thenReturn(RemoteApi.Response.success(200)); + + writer.write(trace); + writer.write(trace); + writer.flush(); + + verify(discovery, times(2)).getTraceEndpoint(); + verify(api, times(1)).sendSerializedTraces(argThat(payload -> payload.traceCount() == 2)); + verifyNoMoreInteractions(api, discovery); + + writer.close(); + } + + @TableTest({ + "scenario | agentVersion ", + "v0.3 | 'v0.3/traces'", + "v0.4 | 'v0.4/traces'", + "v0.5 | 'v0.5/traces'" + }) + void testFloodOfTraces(String agentVersion) { + // bufferSize = 1024; traceCount = 100 (shouldn't trigger payload, but bigger than disruptor + // size) + int bufferSize = 1024; + int traceCount = 100; + + DDAgentApi api = mock(DDAgentApi.class); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + DDAgentWriter writer = + DDAgentWriter.builder() + .featureDiscovery(discovery) + .agentApi(api) + .traceBufferSize(bufferSize) + .monitoring(monitoring) + .flushIntervalMilliseconds(-1) + .build(); + writer.start(); + DDSpan span = (DDSpan) dummyTracer.buildSpan("datadog", "fakeOperation").start(); + List trace = Collections.singletonList(span); + + when(discovery.getTraceEndpoint()).thenReturn(agentVersion); + when(api.sendSerializedTraces(argThat(payload -> payload.traceCount() <= traceCount))) + .thenReturn(RemoteApi.Response.success(200)); + + for (int i = 1; i <= traceCount; i++) { + writer.write(trace); + } + writer.flush(); + + verify(discovery, times(2)).getTraceEndpoint(); + verify(api, times(1)) + .sendSerializedTraces(argThat(payload -> payload.traceCount() <= traceCount)); + verifyNoMoreInteractions(api, discovery); + + writer.close(); + } + + @TableTest({ + "scenario | agentVersion ", + "v0.3 | 'v0.3/traces'", + "v0.4 | 'v0.4/traces'", + "v0.5 | 'v0.5/traces'" + }) + void testFlushByTime(String agentVersion) throws Exception { + HealthMetrics healthMetrics = mock(HealthMetrics.class); + DDAgentApi api = mock(DDAgentApi.class); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + DDAgentWriter writer = + DDAgentWriter.builder() + .featureDiscovery(discovery) + .agentApi(api) + .healthMetrics(healthMetrics) + .monitoring(monitoring) + .flushIntervalMilliseconds(1000) + .build(); + writer.start(); + DDSpan span = (DDSpan) dummyTracer.buildSpan("datadog", "fakeOperation").start(); + List trace = Collections.nCopies(10, span); + + when(discovery.getTraceEndpoint()).thenReturn(agentVersion); + when(api.sendSerializedTraces(argThat(payload -> payload.traceCount() == 5))) + .thenReturn(RemoteApi.Response.success(200)); + + // stub onSend to arrive at phaser + doAnswer( + invocation -> { + phaser.arrive(); + return null; + }) + .when(healthMetrics) + .onSend(anyInt(), anyInt(), any()); + + for (int i = 1; i <= 5; i++) { + writer.write(trace); + } + phaser.awaitAdvanceInterruptibly(phaser.arriveAndDeregister()); + + verify(discovery, times(2)).getTraceEndpoint(); + verify(healthMetrics, times(1)).onSerialize(anyInt()); + verify(api, times(1)).sendSerializedTraces(argThat(payload -> payload.traceCount() == 5)); + // _ * healthMetrics.onPublish(_, _) means any number, so don't verify count + verify(healthMetrics).onSend(anyInt(), anyInt(), any()); + verifyNoMoreInteractions(api, discovery); + + writer.close(); + } + + @Timeout(value = 30, unit = TimeUnit.SECONDS) + @WithConfig(key = EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED, value = "false") + @TableTest({ + "scenario | agentVersion ", + "v0.3 | 'v0.3/traces'", + "v0.4 | 'v0.4/traces'", + "v0.5 | 'v0.5/traces'" + }) + void testDefaultBufferSizeFor(String agentVersion) { + // setup: disable process tags since they are only written on the first span + // and it will break the trace size estimation + List minimalTrace = createMinimalTrace(); + DDAgentApi api = mock(DDAgentApi.class); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + DDAgentWriter writer = + DDAgentWriter.builder() + .featureDiscovery(discovery) + .agentApi(api) + .traceBufferSize(AGENT_WRITER_BUFFER_SIZE) + .prioritization(ENSURE_TRACE) + .monitoring(monitoring) + .flushIntervalMilliseconds(-1) + .build(); + writer.start(); + + TraceMapper mapper = + agentVersion.equals("v0.5/traces") ? new TraceMapperV0_5() : new TraceMapperV0_4(); + int traceSize = calculateSize(minimalTrace, mapper); + int maxedPayloadTraceCount = (mapper.messageBufferSize() / traceSize); + + when(discovery.getTraceEndpoint()).thenReturn(agentVersion); + when(api.sendSerializedTraces( + argThat(payload -> payload != null && payload.traceCount() == maxedPayloadTraceCount))) + .thenReturn(RemoteApi.Response.success(200)); + when(api.sendSerializedTraces(argThat(payload -> payload != null && payload.traceCount() == 1))) + .thenReturn(RemoteApi.Response.success(200)); + + for (int i = 0; i <= maxedPayloadTraceCount; i++) { + writer.write(minimalTrace); + } + writer.flush(); + + verify(discovery, times(2)).getTraceEndpoint(); + verify(api, times(1)) + .sendSerializedTraces( + argThat(payload -> payload != null && payload.traceCount() == maxedPayloadTraceCount)); + verify(api, times(1)) + .sendSerializedTraces(argThat(payload -> payload != null && payload.traceCount() == 1)); + verifyNoMoreInteractions(api, discovery); + + writer.close(); + } + + @Test + void checkThatThereAreNoInteractionsAfterClose() { + HealthMetrics healthMetrics = mock(HealthMetrics.class); + DDAgentApi api = mock(DDAgentApi.class); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + DDAgentWriter writer = + DDAgentWriter.builder() + .featureDiscovery(discovery) + .agentApi(api) + .healthMetrics(healthMetrics) + .monitoring(monitoring) + .build(); + writer.start(); + // Clear invocations from start() (Spock only counts interactions in when: blocks) + clearInvocations(healthMetrics, api, discovery); + + writer.close(); + writer.write(Collections.emptyList()); + writer.flush(); + + // then: this will be checked during flushing + verify(healthMetrics, times(1)).onFailedPublish(anyInt(), anyInt()); + verify(healthMetrics, times(1)).onFlush(any(Boolean.class)); + verify(healthMetrics, times(1)).onShutdown(any(Boolean.class)); + verify(healthMetrics, times(1)).close(); + verifyNoMoreInteractions(healthMetrics, api, discovery); + + writer.close(); + } + + @TableTest({ + "scenario | agentVersion ", + "v0.3 | 'v0.3/traces'", + "v0.4 | 'v0.4/traces'", + "v0.5 | 'v0.5/traces'" + }) + void monitorHappyPath(String agentVersion) { + HealthMetrics healthMetrics = mock(HealthMetrics.class); + List minimalTrace = createMinimalTrace(); + + // DQH -- need to set-up a dummy agent for the final send callback to work + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + server -> + server.handlers( + h -> h.put(agentVersion, api -> api.getResponse().status(200).send()))); + try { + HttpUrl agentUrl = HttpUrl.get(agent.getAddress()); + okhttp3.OkHttpClient client = OkHttpUtils.buildHttpClient(agentUrl, 1000); + DDAgentFeaturesDiscovery discovery = + new DDAgentFeaturesDiscovery(client, monitoring, agentUrl, V0_5, true, false); + DDAgentApi api = new DDAgentApi(client, agentUrl, discovery, monitoring, true); + DDAgentWriter writer = + DDAgentWriter.builder() + .featureDiscovery(discovery) + .agentApi(api) + .monitoring(monitoring) + .healthMetrics(healthMetrics) + .build(); + + // start + writer.start(); + + verify(healthMetrics, times(1)).onStart((int) writer.getCapacity()); + + // write and flush + writer.write(minimalTrace); + writer.flush(); + + verify(healthMetrics, times(1)).onPublish(any(), anyInt()); + verify(healthMetrics, times(1)).onSerialize(anyInt()); + verify(healthMetrics, times(1)).onFlush(false); + verify(healthMetrics, times(1)) + .onSend( + anyInt(), + anyInt(), + argThat( + response -> + response.success() + && response.status().isPresent() + && response.status().getAsInt() == 200)); + + writer.close(); + + verify(healthMetrics, times(1)).onShutdown(true); + } finally { + agent.close(); + } + } + + @TableTest({ + "scenario | agentVersion ", + "v0.3 | 'v0.3/traces'", + "v0.4 | 'v0.4/traces'", + "v0.5 | 'v0.5/traces'" + }) + void monitorAgentReturnsError(String agentVersion) { + HealthMetrics healthMetrics = mock(HealthMetrics.class); + List minimalTrace = createMinimalTrace(); + + // DQH -- need to set-up a dummy agent for the final send callback to work + final boolean[] first = {true}; + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + server -> + server.handlers( + h -> + h.put( + agentVersion, + api -> { + // DQH - DDApi sniffs for end point existence, so respond with 200 the + // first time + if (first[0]) { + api.getResponse().status(200).send(); + first[0] = false; + } else { + api.getResponse().status(500).send(); + } + }))); + try { + HttpUrl agentUrl = HttpUrl.get(agent.getAddress()); + okhttp3.OkHttpClient client = OkHttpUtils.buildHttpClient(agentUrl, 1000); + DDAgentFeaturesDiscovery discovery = + new DDAgentFeaturesDiscovery(client, monitoring, agentUrl, V0_5, true, false); + DDAgentApi api = new DDAgentApi(client, agentUrl, discovery, monitoring, true); + DDAgentWriter writer = + DDAgentWriter.builder() + .featureDiscovery(discovery) + .agentApi(api) + .monitoring(monitoring) + .healthMetrics(healthMetrics) + .build(); + + // start + writer.start(); + + verify(healthMetrics, times(1)).onStart((int) writer.getCapacity()); + + // write and flush + writer.write(minimalTrace); + writer.flush(); + + verify(healthMetrics, times(1)).onPublish(any(), anyInt()); + verify(healthMetrics, times(1)).onSerialize(anyInt()); + verify(healthMetrics, times(1)).onFlush(false); + verify(healthMetrics, times(1)) + .onFailedSend( + anyInt(), + anyInt(), + argThat( + response -> + !response.success() + && response.status().isPresent() + && response.status().getAsInt() == 500)); + + writer.close(); + + verify(healthMetrics, times(1)).onShutdown(true); + } finally { + agent.close(); + } + } + + @TableTest({ + "scenario | agentVersion ", + "v0.3 | 'v0.3/traces'", + "v0.4 | 'v0.4/traces'", + "v0.5 | 'v0.5/traces'" + }) + void unreachableAgentTest(String agentVersion) { + HealthMetrics healthMetrics = mock(HealthMetrics.class); + List minimalTrace = createMinimalTrace(); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + when(discovery.getTraceEndpoint()).thenReturn(agentVersion); + DDAgentApi api = mock(DDAgentApi.class); + // simulating a communication failure to a server + when(api.sendSerializedTraces(any())) + .thenReturn(RemoteApi.Response.failed(new IOException("comm error"))); + + DDAgentWriter writer = + DDAgentWriter.builder() + .featureDiscovery(discovery) + .agentApi(api) + .monitoring(monitoring) + .healthMetrics(healthMetrics) + .build(); + + // start + writer.start(); + + verify(healthMetrics, times(1)).onStart((int) writer.getCapacity()); + + // write and flush + writer.write(minimalTrace); + writer.flush(); + + // then: if we know there's no agent, we'll drop the traces before serialising them + // but we also know that there's nowhere to send health metrics to + verify(healthMetrics, times(1)).onPublish(any(), anyInt()); + verify(healthMetrics, times(1)).onFlush(false); + + writer.close(); + + verify(healthMetrics, times(1)).onShutdown(true); + } + + @Flaky("If execution is too slow, the http client timeout may trigger") + @TableTest({ + "scenario | agentVersion ", + "v0.3 | 'v0.3/traces'", + "v0.4 | 'v0.4/traces'", + "v0.5 | 'v0.5/traces'" + }) + void slowResponseTest(String agentVersion) throws Exception { + int numWritten = 0; + AtomicInteger numFlushes = new AtomicInteger(0); + AtomicInteger numPublished = new AtomicInteger(0); + AtomicInteger numFailedPublish = new AtomicInteger(0); + AtomicInteger numRequests = new AtomicInteger(0); + AtomicInteger numFailedRequests = new AtomicInteger(0); + + Semaphore responseSemaphore = new Semaphore(1); + + // Need to set-up a dummy agent for the final send callback to work + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + server -> + server.handlers( + h -> + h.put( + agentVersion, + api -> { + responseSemaphore.acquire(); + try { + api.getResponse().status(200).send(); + } finally { + responseSemaphore.release(); + } + }))); + + // This test focuses just on failed publish, so not verifying every callback + HealthMetrics healthMetrics = mock(HealthMetrics.class); + doAnswer( + invocation -> { + numPublished.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onPublish(any(), anyInt()); + doAnswer( + invocation -> { + numFailedPublish.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onFailedPublish(anyInt(), anyInt()); + doAnswer( + invocation -> { + numFlushes.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onFlush(any(Boolean.class)); + doAnswer( + invocation -> { + numRequests.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onSend(anyInt(), anyInt(), any()); + doAnswer( + invocation -> { + numFailedRequests.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onFailedSend(anyInt(), anyInt(), any()); + + int bufferSize = 16; + List minimalTrace = createMinimalTrace(); + DDAgentWriter writer = + DDAgentWriter.builder() + .traceAgentProtocolVersion(V0_5) + .traceAgentPort(agent.getAddress().getPort()) + .monitoring(monitoring) + .healthMetrics(healthMetrics) + .traceBufferSize(bufferSize) + .build(); + writer.start(); + + // gate responses + responseSemaphore.acquire(); + + try { + // when: write a single trace and flush + // with responseSemaphore held, the response is blocked but may still time out + writer.write(minimalTrace); + numWritten += 1; + + // sanity check coordination mechanism of test + // release to allow response to be generated + responseSemaphore.release(); + writer.flush(); + + // reacquire semaphore to stall further responses + responseSemaphore.acquire(); + + assertEquals(0, numFailedPublish.get()); + assertEquals(numWritten, numPublished.get()); + assertEquals(numWritten, numPublished.get() + numFailedPublish.get()); + assertEquals(1, numFlushes.get()); + + // when: send many traces to fill the sender queue... + // loop until outstanding requests > finished requests + while (writer.traceProcessingWorker.getRemainingCapacity() > 0 + || numFailedPublish.get() == 0) { + writer.write(minimalTrace); + numWritten += 1; + } + + assertTrue(numFailedPublish.get() > 0); + assertEquals(numWritten, numPublished.get() + numFailedPublish.get()); + + // with both disruptor & queue full, should reject everything + int expectedRejects = 100; + for (int i = 1; i <= expectedRejects; i++) { + writer.write(minimalTrace); + numWritten += 1; + } + + assertEquals(numWritten, numPublished.get() + numFailedPublish.get()); + } finally { + responseSemaphore.release(); + writer.close(); + agent.close(); + } + } + + @TableTest({ + "scenario | agentVersion ", + "v0.3 | 'v0.3/traces'", + "v0.4 | 'v0.4/traces'", + "v0.5 | 'v0.5/traces'" + }) + void multiThreaded(String agentVersion) throws Exception { + AtomicInteger numPublished = new AtomicInteger(0); + AtomicInteger numFailedPublish = new AtomicInteger(0); + AtomicInteger numRepSent = new AtomicInteger(0); + + List minimalTrace = createMinimalTrace(); + + // Need to set-up a dummy agent for the final send callback to work + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + server -> + server.handlers( + h -> h.put(agentVersion, api -> api.getResponse().status(200).send()))); + + // This test focuses just on failed publish, so not verifying every callback + HealthMetrics healthMetrics = mock(HealthMetrics.class); + doAnswer( + invocation -> { + numPublished.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onPublish(any(), anyInt()); + doAnswer( + invocation -> { + numFailedPublish.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onFailedPublish(anyInt(), anyInt()); + doAnswer( + invocation -> { + int repCount = invocation.getArgument(0); + numRepSent.addAndGet(repCount); + return null; + }) + .when(healthMetrics) + .onSend(anyInt(), anyInt(), any()); + + DDAgentWriter writer = + DDAgentWriter.builder() + .traceAgentProtocolVersion(V0_5) + .traceAgentPort(agent.getAddress().getPort()) + .monitoring(monitoring) + .healthMetrics(healthMetrics) + .build(); + writer.start(); + + try { + Runnable producer = + () -> { + for (int i = 1; i <= 100; i++) { + writer.write(minimalTrace); + } + }; + + Thread t1 = new Thread(producer); + t1.start(); + + Thread t2 = new Thread(producer); + t2.start(); + + t1.join(); + t2.join(); + + writer.flush(); + + // then: conditions.eventually { assert numPublished.get() == 200 && numRepSent.get() == 200 } + int totalTraces = 100 + 100; + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + if (numPublished.get() == totalTraces && numRepSent.get() == totalTraces) { + break; + } + Thread.sleep(50); + } + assertEquals(totalTraces, numPublished.get()); + assertEquals(totalTraces, numRepSent.get()); + } finally { + writer.close(); + agent.close(); + } + } + + @TableTest({ + "scenario | agentVersion ", + "v0.3 | 'v0.3/traces'", + "v0.4 | 'v0.4/traces'", + "v0.5 | 'v0.5/traces'" + }) + void statsdSuccess(String agentVersion) { + AtomicInteger numTracesAccepted = new AtomicInteger(0); + AtomicInteger numRequests = new AtomicInteger(0); + AtomicInteger numResponses = new AtomicInteger(0); + + List minimalTrace = createMinimalTrace(); + + // Need to set-up a dummy agent for the final send callback to work + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + server -> + server.handlers( + h -> h.put(agentVersion, api -> api.getResponse().status(200).send()))); + + HealthMetrics healthMetrics = mock(HealthMetrics.class); + doAnswer( + invocation -> { + numTracesAccepted.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onPublish(any(), anyInt()); + doAnswer( + invocation -> { + numRequests.incrementAndGet(); + numResponses.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onSend(anyInt(), anyInt(), any()); + DDAgentWriter writer = + DDAgentWriter.builder() + .agentHost(agent.getAddress().getHost()) + .traceAgentProtocolVersion(V0_5) + .traceAgentPort(agent.getAddress().getPort()) + .monitoring(monitoring) + .healthMetrics(healthMetrics) + .build(); + writer.start(); + + try { + writer.write(minimalTrace); + writer.flush(); + + assertEquals(1, numTracesAccepted.get()); + assertEquals(1, numRequests.get()); + assertEquals(1, numResponses.get()); + } finally { + agent.close(); + writer.close(); + } + } + + @Test + void statsdCommFailure() throws Exception { + List minimalTrace = createMinimalTrace(); + + DDAgentApi api = Mockito.mock(DDAgentApi.class); + when(api.sendSerializedTraces(any())) + .thenReturn(RemoteApi.Response.failed(new IOException("comm error"))); + + CountDownLatch latch = new CountDownLatch(2); + StatsDClient statsd = mock(StatsDClient.class); + TracerHealthMetrics healthMetrics = new TracerHealthMetrics(statsd, 100, TimeUnit.MILLISECONDS); + DDAgentWriter writer = + DDAgentWriter.builder() + .traceAgentProtocolVersion(V0_5) + .agentApi(api) + .monitoring(monitoring) + .healthMetrics(healthMetrics) + .build(); + healthMetrics.start(); + writer.start(); + + // stub statsd.count for latch coordination - called with varargs String... tags + doAnswer( + invocation -> { + latch.countDown(); + return null; + }) + .when(statsd) + .count(anyString(), anyLong()); + + writer.write(minimalTrace); + writer.flush(); + latch.await(10, TimeUnit.SECONDS); + + verify(statsd, times(1)).count("api.requests.total", 1L); + verify(statsd, times(0)).incrementCounter("api.responses.total"); + verify(statsd, times(1)).count("api.errors.total", 1L); + + writer.close(); + healthMetrics.close(); + } + + static int calculateSize(List trace, TraceMapper mapper) { + AtomicInteger size = new AtomicInteger(); + MsgPackWriter packer = + new MsgPackWriter( + new FlushingBuffer( + 1024, (messageCount, buffer) -> size.set(buffer.limit() - buffer.position()))); + packer.format(trace, mapper); + packer.flush(); + return size.get(); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterTest.java new file mode 100644 index 00000000000..8aeea35d6f2 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterTest.java @@ -0,0 +1,226 @@ +package datadog.trace.common.writer; + +import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.ENQUEUED_FOR_SERIALIZATION; +import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.ENQUEUED_FOR_SINGLE_SPAN_SAMPLING; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.metrics.api.statsd.StatsDClient; +import datadog.metrics.impl.MonitoringImpl; +import datadog.trace.common.writer.ddagent.DDAgentApi; +import datadog.trace.common.writer.ddagent.DDAgentMapperDiscovery; +import datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult; +import datadog.trace.core.CoreTracer; +import datadog.trace.core.DDCoreJavaSpecification; +import datadog.trace.core.DDSpan; +import datadog.trace.core.monitor.HealthMetrics; +import datadog.trace.core.propagation.PropagationTags; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.tabletest.junit.TableTest; + +class DDAgentWriterTest extends DDCoreJavaSpecification { + + HealthMetrics monitor = mock(HealthMetrics.class); + TraceProcessingWorker worker = mock(TraceProcessingWorker.class); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + DDAgentApi api = mock(DDAgentApi.class); + MonitoringImpl monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS); + PayloadDispatcherImpl dispatcher = + new PayloadDispatcherImpl(new DDAgentMapperDiscovery(discovery), api, monitor, monitoring); + DDAgentWriter writer = new DDAgentWriter(worker, dispatcher, monitor, 1, TimeUnit.SECONDS, false); + + // Only used to create spans + CoreTracer dummyTracer; + + @BeforeEach + void setup() { + dummyTracer = tracerBuilder().writer(new ListWriter()).build(); + } + + @AfterEach + void cleanup() { + writer.close(); + if (dummyTracer != null) { + dummyTracer.close(); + } + } + + @Test + void testWriterBuilder() { + DDAgentWriter builtWriter = DDAgentWriter.builder().build(); + + assertNotNull(builtWriter); + } + + @Test + void testWriterStart() { + int capacity = 5; + + when(worker.getCapacity()).thenReturn(capacity); + writer.start(); + + verify(monitor).start(); + verify(worker).start(); + verify(worker).getCapacity(); + verify(monitor).onStart(capacity); + verifyNoMoreInteractions(monitor, worker, discovery, api); + } + + @Test + void testWriterStartClosed() { + writer.close(); + clearInvocations(monitor, worker, discovery, api); + + writer.start(); + + verifyNoMoreInteractions(monitor, worker, discovery, api); + } + + @Test + void testWriterFlush() { + when(worker.flush(1, TimeUnit.SECONDS)).thenReturn(true, false); + + // first flush succeeds + writer.flush(); + + // monitor is notified + verify(worker).flush(1, TimeUnit.SECONDS); + verify(monitor).onFlush(false); + verifyNoMoreInteractions(monitor, worker, discovery, api); + + clearInvocations(monitor, worker, discovery, api); + + // second flush returns false + writer.flush(); + + // no additional monitor notifications + verify(worker).flush(1, TimeUnit.SECONDS); + verifyNoMoreInteractions(monitor, worker, discovery, api); + } + + @Test + void testWriterFlushClosed() { + writer.close(); + clearInvocations(monitor, worker, discovery, api); + + writer.flush(); + + verifyNoMoreInteractions(monitor, worker, discovery, api); + } + + @Test + void testWriterWritePublishSucceeds() { + List trace = + Collections.singletonList( + (DDSpan) dummyTracer.buildSpan("datadog", "fakeOperation").start()); + + // publish succeeds + when(worker.publish(any(), anyInt(), eq(trace))).thenReturn(ENQUEUED_FOR_SERIALIZATION); + writer.write(trace); + + // monitor is notified of successful publication + verify(worker).publish(any(), anyInt(), eq(trace)); + verify(monitor).onPublish(any(), anyInt()); + verifyNoMoreInteractions(monitor, worker, discovery, api); + } + + @Test + void testWriterWritePublishForSingleSpanSampling() { + List trace = + Collections.singletonList( + (DDSpan) dummyTracer.buildSpan("datadog", "fakeOperation").start()); + + // publish succeeds (single span sampling) + when(worker.publish(any(), anyInt(), eq(trace))).thenReturn(ENQUEUED_FOR_SINGLE_SPAN_SAMPLING); + writer.write(trace); + + // monitor should not call onPublish for single span sampling + verify(worker).publish(any(), anyInt(), eq(trace)); + verifyNoMoreInteractions(monitor, worker, discovery, api); + } + + @TableTest({ + "scenario | publishResult ", + "buffer overflow | DROPPED_BUFFER_OVERFLOW", + "dropped by policy | DROPPED_BY_POLICY " + }) + void testWriterWritePublishFails(PublishResult publishResult) { + List trace = + Collections.singletonList( + (DDSpan) dummyTracer.buildSpan("datadog", "fakeOperation").start()); + + // publish fails + when(worker.publish(any(), anyInt(), eq(trace))).thenReturn(publishResult); + writer.write(trace); + + // monitor is notified of unsuccessful publication + verify(worker).publish(any(), anyInt(), eq(trace)); + verify(monitor).onFailedPublish(anyInt(), anyInt()); + verifyNoMoreInteractions(monitor, worker, discovery, api); + } + + @Test + void testEmptyTracesShouldBeReportedAsFailures() { + // trace is empty + writer.write(Collections.emptyList()); + + // monitor is notified of unsuccessful publication + verify(monitor).onFailedPublish(anyInt(), anyInt()); + verifyNoMoreInteractions(monitor, worker, discovery, api); + } + + @Test + void testWriterWriteClosed() { + writer.close(); + clearInvocations(monitor, worker, discovery, api); + List trace = + Collections.singletonList( + (DDSpan) dummyTracer.buildSpan("datadog", "fakeOperation").start()); + + writer.write(trace); + + verify(monitor).onFailedPublish(anyInt(), anyInt()); + verifyNoMoreInteractions(monitor, worker, discovery, api); + } + + @TableTest({ + "scenario | publishResult ", + "dropped by policy | DROPPED_BY_POLICY ", + "buffer overflow | DROPPED_BUFFER_OVERFLOW" + }) + void testDroppedTraceIsCounted(PublishResult publishResult) { + // setup - use local mocks to avoid interference with instance-level mocks + TraceProcessingWorker localWorker = mock(TraceProcessingWorker.class); + HealthMetrics localMonitor = mock(HealthMetrics.class); + PayloadDispatcherImpl localDispatcher = mock(PayloadDispatcherImpl.class); + DDAgentWriter localWriter = + new DDAgentWriter(localWorker, localDispatcher, localMonitor, 1, TimeUnit.SECONDS, false); + + DDSpan p0 = newSpan(); + List trace = java.util.Arrays.asList(p0, newSpan()); + + when(localWorker.publish(eq(trace.get(0)), anyInt(), eq(trace))).thenReturn(publishResult); + localWriter.write(trace); + + verify(localWorker).publish(eq(trace.get(0)), anyInt(), eq(trace)); + verify(localDispatcher).onDroppedTrace(trace.size()); + } + + DDSpan newSpan() { + // Use the UNSET-priority variant so setSamplingPriority() can change the priority later + return buildSpan(0L, "test.tag", "test.value", PropagationTags.factory().empty()); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterCombinedTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterCombinedTest.java new file mode 100644 index 00000000000..56883149858 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterCombinedTest.java @@ -0,0 +1,825 @@ +package datadog.trace.common.writer; + +import static datadog.trace.common.writer.ddagent.Prioritization.ENSURE_TRACE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +import datadog.communication.http.OkHttpUtils; +import datadog.communication.serialization.FlushingBuffer; +import datadog.communication.serialization.msgpack.MsgPackWriter; +import datadog.metrics.api.statsd.StatsDClient; +import datadog.metrics.impl.MonitoringImpl; +import datadog.trace.agent.test.server.http.JavaTestHttpServer; +import datadog.trace.api.civisibility.CiVisibilityWellKnownTags; +import datadog.trace.api.intake.TrackType; +import datadog.trace.common.writer.ddintake.DDIntakeApi; +import datadog.trace.common.writer.ddintake.DDIntakeMapperDiscovery; +import datadog.trace.core.DDCoreJavaSpecification; +import datadog.trace.core.DDSpan; +import datadog.trace.core.monitor.HealthMetrics; +import datadog.trace.core.monitor.TracerHealthMetrics; +import datadog.trace.test.util.Flaky; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Phaser; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import okhttp3.HttpUrl; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(value = 10, unit = TimeUnit.SECONDS) +class DDIntakeWriterCombinedTest extends DDCoreJavaSpecification { + + private static final CiVisibilityWellKnownTags wellKnownTags = + new CiVisibilityWellKnownTags( + "my-runtime-id", + "my-env", + "my-language", + "my-runtime-name", + "my-runtime-version", + "my-runtime-vendor", + "my-os-arch", + "my-os-platform", + "my-os-version", + "false"); + + MonitoringImpl monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS); + Phaser phaser = new Phaser(); + + // Only used to create spans + datadog.trace.core.CoreTracer dummyTracer; + + @BeforeEach + void setup() { + // Register for two threads. + phaser.register(); + phaser.register(); + dummyTracer = tracerBuilder().writer(new ListWriter()).build(); + } + + @AfterEach + void cleanup() { + if (dummyTracer != null) { + dummyTracer.close(); + } + } + + List createMinimalTrace() { + // Use buildSpan from DDCoreJavaSpecification to create a real DDSpan with minimal fields + DDSpan span = buildSpan(0L, "", Collections.emptyMap()); + return Collections.singletonList(span); + } + + @Test + void noInteractionsBecauseOfInitialFlush() { + DDIntakeApi api = mock(DDIntakeApi.class); + DDIntakeWriter writer = + DDIntakeWriter.builder() + .addTrack(TrackType.NOOP, api) + .traceBufferSize(8) + .monitoring(monitoring) + .flushIntervalMilliseconds(-1) + .alwaysFlush(false) + .build(); + writer.start(); + // Clear setup-time interactions (e.g. isCompressionEnabled() called during build()) + clearInvocations(api); + + writer.flush(); + + // then: 0 * _ (no interactions at all on mocked api) + verifyNoMoreInteractions(api); + + writer.close(); + } + + @Test + void testHappyPath() { + TrackType trackType = TrackType.CITESTCYCLE; + DDIntakeApi api = mock(DDIntakeApi.class); + DDIntakeWriter writer = + DDIntakeWriter.builder() + .addTrack(trackType, api) + .traceBufferSize(1024) + .monitoring(monitoring) + .flushIntervalMilliseconds(-1) + .alwaysFlush(false) + .build(); + writer.start(); + // Clear setup-time interactions (e.g. isCompressionEnabled() called during build()) + clearInvocations(api); + DDSpan span = (DDSpan) dummyTracer.buildSpan("datadog", "fakeOperation").start(); + List trace = Collections.singletonList(span); + + doAnswer(invocation -> RemoteApi.Response.success(200)) + .when(api) + .sendSerializedTraces(argThat(payload -> payload.traceCount() == 2)); + + writer.write(trace); + writer.write(trace); + writer.flush(); + + verify(api, times(1)).sendSerializedTraces(argThat(payload -> payload.traceCount() == 2)); + verifyNoMoreInteractions(api); + + writer.close(); + } + + @Test + void testFloodOfTraces() { + // bufferSize = 1024; traceCount = 100 (shouldn't trigger payload, but bigger than disruptor + // size) + int traceCount = 100; + TrackType trackType = TrackType.CITESTCYCLE; + DDIntakeApi api = mock(DDIntakeApi.class); + DDIntakeWriter writer = + DDIntakeWriter.builder() + .addTrack(trackType, api) + .traceBufferSize(1024) + .monitoring(monitoring) + .flushIntervalMilliseconds(-1) + .alwaysFlush(false) + .build(); + writer.start(); + // Clear setup-time interactions (e.g. isCompressionEnabled() called during build()) + clearInvocations(api); + DDSpan span = (DDSpan) dummyTracer.buildSpan("datadog", "fakeOperation").start(); + List trace = Collections.singletonList(span); + + doAnswer(invocation -> RemoteApi.Response.success(200)) + .when(api) + .sendSerializedTraces(argThat(payload -> payload.traceCount() <= traceCount)); + + for (int i = 1; i <= traceCount; i++) { + writer.write(trace); + } + writer.flush(); + + verify(api, times(1)) + .sendSerializedTraces(argThat(payload -> payload.traceCount() <= traceCount)); + verifyNoMoreInteractions(api); + + writer.close(); + } + + @Test + void testFlushByTime() throws Exception { + TrackType trackType = TrackType.CITESTCYCLE; + HealthMetrics healthMetrics = mock(HealthMetrics.class); + DDIntakeApi api = mock(DDIntakeApi.class); + DDIntakeWriter writer = + DDIntakeWriter.builder() + .addTrack(trackType, api) + .healthMetrics(healthMetrics) + .monitoring(monitoring) + .flushIntervalMilliseconds(1000) + .alwaysFlush(false) + .build(); + writer.start(); + // Clear setup-time interactions (e.g. isCompressionEnabled() called during build(), onStart + // from start()) + clearInvocations(api, healthMetrics); + DDSpan span = (DDSpan) dummyTracer.buildSpan("datadog", "fakeOperation").start(); + List trace = Collections.nCopies(10, span); + + doAnswer(invocation -> RemoteApi.Response.success(200)) + .when(api) + .sendSerializedTraces(argThat(payload -> payload.traceCount() == 5)); + + // stub onSend to arrive at phaser + doAnswer( + invocation -> { + phaser.arrive(); + return null; + }) + .when(healthMetrics) + .onSend(anyInt(), anyInt(), any()); + + for (int i = 1; i <= 5; i++) { + writer.write(trace); + } + phaser.awaitAdvanceInterruptibly(phaser.arriveAndDeregister()); + + verify(healthMetrics, times(1)).onSerialize(anyInt()); + verify(api, times(1)).sendSerializedTraces(argThat(payload -> payload.traceCount() == 5)); + // _ * healthMetrics.onPublish(_, _) means any number, so don't verify count + verify(healthMetrics).onSend(anyInt(), anyInt(), any()); + verifyNoMoreInteractions(api); + + writer.close(); + } + + @Timeout(value = 30, unit = TimeUnit.SECONDS) + @Test + void testDefaultBufferSizeForCitestcycle() { + TrackType trackType = TrackType.CITESTCYCLE; + List minimalTrace = createMinimalTrace(); + DDIntakeApi api = mock(DDIntakeApi.class); + DDIntakeWriter writer = + DDIntakeWriter.builder() + .addTrack(trackType, api) + .wellKnownTags(wellKnownTags) + .traceBufferSize(1024) + .prioritization(ENSURE_TRACE) + .monitoring(monitoring) + .flushIntervalMilliseconds(-1) + .alwaysFlush(false) + .build(); + writer.start(); + // Clear setup-time interactions (e.g. isCompressionEnabled() called during build()) + clearInvocations(api); + + DDIntakeMapperDiscovery discovery = + new DDIntakeMapperDiscovery(trackType, wellKnownTags, false); + discovery.discover(); + RemoteMapper mapper = discovery.getMapper(); + int traceSize = calculateSize(minimalTrace, mapper); + int maxedPayloadTraceCount = (mapper.messageBufferSize() / traceSize); + + doAnswer(invocation -> RemoteApi.Response.success(200)) + .when(api) + .sendSerializedTraces( + argThat(payload -> payload != null && payload.traceCount() == maxedPayloadTraceCount)); + doAnswer(invocation -> RemoteApi.Response.success(200)) + .when(api) + .sendSerializedTraces(argThat(payload -> payload != null && payload.traceCount() == 1)); + + for (int i = 0; i <= maxedPayloadTraceCount; i++) { + writer.write(minimalTrace); + } + writer.flush(); + + verify(api, times(1)) + .sendSerializedTraces( + argThat(payload -> payload != null && payload.traceCount() == maxedPayloadTraceCount)); + verify(api, times(1)) + .sendSerializedTraces(argThat(payload -> payload != null && payload.traceCount() == 1)); + verifyNoMoreInteractions(api); + + writer.close(); + } + + @Test + void checkThatThereAreNoInteractionsAfterClose() { + TrackType trackType = TrackType.CITESTCYCLE; + DDIntakeApi api = mock(DDIntakeApi.class); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + DDIntakeWriter writer = + DDIntakeWriter.builder() + .addTrack(trackType, api) + .healthMetrics(healthMetrics) + .monitoring(monitoring) + .alwaysFlush(false) + .build(); + writer.start(); + // Clear setup-time interactions (e.g. isCompressionEnabled() called during build(), onStart + // from start()) + clearInvocations(api, healthMetrics); + + writer.close(); + writer.write(Collections.emptyList()); + writer.flush(); + + // then: this will be checked during flushing + verify(healthMetrics, times(1)).onFailedPublish(anyInt(), anyInt()); + verify(healthMetrics, times(1)).onFlush(any(Boolean.class)); + verify(healthMetrics, times(1)).onShutdown(any(Boolean.class)); + verify(healthMetrics, times(1)).close(); + verifyNoMoreInteractions(healthMetrics, api); + + writer.close(); + } + + @Test + void monitorHappyPath() { + TrackType trackType = TrackType.CITESTCYCLE; + String apiVersion = "v2"; + HealthMetrics healthMetrics = mock(HealthMetrics.class); + List minimalTrace = createMinimalTrace(); + String path = buildIntakePath(trackType, apiVersion); + + JavaTestHttpServer intake = + JavaTestHttpServer.httpServer( + server -> + server.handlers(h -> h.post(path, api -> api.getResponse().status(200).send()))); + try { + HttpUrl hostUrl = HttpUrl.get(intake.getAddress()); + okhttp3.OkHttpClient httpClient = OkHttpUtils.buildHttpClient(hostUrl, 1000); + DDIntakeApi api = + DDIntakeApi.builder() + .hostUrl(hostUrl) + .httpClient(httpClient) + .apiKey("my-api-key") + .trackType(trackType) + .build(); + DDIntakeWriter writer = + DDIntakeWriter.builder() + .addTrack(trackType, api) + .healthMetrics(healthMetrics) + .monitoring(monitoring) + .alwaysFlush(false) + .build(); + + // start + writer.start(); + + verify(healthMetrics, times(1)).onStart((int) writer.getCapacity()); + + // write and flush + writer.write(minimalTrace); + writer.flush(); + + verify(healthMetrics, times(1)).onPublish(any(), anyInt()); + verify(healthMetrics, times(1)).onSerialize(anyInt()); + verify(healthMetrics, times(1)).onFlush(false); + verify(healthMetrics, times(1)) + .onSend( + anyInt(), + anyInt(), + argThat( + response -> + response.success() + && response.status().isPresent() + && response.status().getAsInt() == 200)); + + writer.close(); + + verify(healthMetrics, times(1)).onShutdown(true); + } finally { + intake.close(); + } + } + + @Test + void monitorIntakeReturnsError() { + TrackType trackType = TrackType.CITESTCYCLE; + String apiVersion = "v2"; + HealthMetrics healthMetrics = mock(HealthMetrics.class); + List minimalTrace = createMinimalTrace(); + String path = buildIntakePath(trackType, apiVersion); + + JavaTestHttpServer intake = + JavaTestHttpServer.httpServer( + server -> + server.handlers(h -> h.post(path, api -> api.getResponse().status(500).send()))); + try { + HttpUrl hostUrl = HttpUrl.get(intake.getAddress()); + okhttp3.OkHttpClient httpClient = OkHttpUtils.buildHttpClient(hostUrl, 1000); + DDIntakeApi api = + DDIntakeApi.builder() + .hostUrl(hostUrl) + .httpClient(httpClient) + .apiKey("my-api-key") + .trackType(trackType) + .build(); + DDIntakeWriter writer = + DDIntakeWriter.builder() + .addTrack(trackType, api) + .healthMetrics(healthMetrics) + .monitoring(monitoring) + .alwaysFlush(false) + .build(); + + // start + writer.start(); + + verify(healthMetrics, times(1)).onStart((int) writer.getCapacity()); + + // write and flush + writer.write(minimalTrace); + writer.flush(); + + verify(healthMetrics, times(1)).onPublish(any(), anyInt()); + verify(healthMetrics, times(1)).onSerialize(anyInt()); + verify(healthMetrics, times(1)).onFlush(false); + verify(healthMetrics, times(1)) + .onFailedSend( + anyInt(), + anyInt(), + argThat( + response -> + !response.success() + && response.status().isPresent() + && response.status().getAsInt() == 500)); + + writer.close(); + + verify(healthMetrics, times(1)).onShutdown(true); + } finally { + intake.close(); + } + } + + @Test + void unreachableIntakeTest() { + TrackType trackType = TrackType.CITESTCYCLE; + HealthMetrics healthMetrics = mock(HealthMetrics.class); + List minimalTrace = createMinimalTrace(); + DDIntakeApi api = mock(DDIntakeApi.class); + // simulating a communication failure to a server + doAnswer(invocation -> RemoteApi.Response.failed(new IOException("comm error"))) + .when(api) + .sendSerializedTraces(any()); + + DDIntakeWriter writer = + DDIntakeWriter.builder() + .addTrack(trackType, api) + .monitoring(monitoring) + .healthMetrics(healthMetrics) + .alwaysFlush(false) + .build(); + + // start + writer.start(); + + verify(healthMetrics, times(1)).onStart((int) writer.getCapacity()); + + // write and flush + writer.write(minimalTrace); + writer.flush(); + + // then: if we know there's no agent, we'll drop the traces before serialising them + // but we also know that there's nowhere to send health metrics to + verify(healthMetrics, times(1)).onPublish(any(), anyInt()); + verify(healthMetrics, times(1)).onFlush(false); + + writer.close(); + + verify(healthMetrics, times(1)).onShutdown(true); + } + + @Flaky("If execution is too slow, the http client timeout may trigger") + @Test + void slowResponseTest() throws Exception { + int numWritten = 0; + AtomicInteger numFlushes = new AtomicInteger(0); + AtomicInteger numPublished = new AtomicInteger(0); + AtomicInteger numFailedPublish = new AtomicInteger(0); + AtomicInteger numRequests = new AtomicInteger(0); + AtomicInteger numFailedRequests = new AtomicInteger(0); + + Semaphore responseSemaphore = new Semaphore(1); + + TrackType trackType = TrackType.CITESTCYCLE; + String apiVersion = "v2"; + int bufferSize = 16; + List minimalTrace = createMinimalTrace(); + String path = buildIntakePath(trackType, apiVersion); + + JavaTestHttpServer intake = + JavaTestHttpServer.httpServer( + server -> + server.handlers( + h -> + h.post( + path, + api -> { + responseSemaphore.acquire(); + try { + api.getResponse().status(200).send(); + } finally { + responseSemaphore.release(); + } + }))); + + // This test focuses just on failed publish, so not verifying every callback + HealthMetrics healthMetrics = mock(HealthMetrics.class); + doAnswer( + invocation -> { + numPublished.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onPublish(any(), anyInt()); + doAnswer( + invocation -> { + numFailedPublish.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onFailedPublish(anyInt(), anyInt()); + doAnswer( + invocation -> { + numFlushes.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onFlush(any(Boolean.class)); + doAnswer( + invocation -> { + numRequests.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onSend(anyInt(), anyInt(), any()); + doAnswer( + invocation -> { + numFailedRequests.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onFailedSend(anyInt(), anyInt(), any()); + + HttpUrl hostUrl = HttpUrl.get(intake.getAddress()); + okhttp3.OkHttpClient httpClient = OkHttpUtils.buildHttpClient(hostUrl, 1000); + DDIntakeApi api = + DDIntakeApi.builder() + .hostUrl(hostUrl) + .httpClient(httpClient) + .apiKey("my-api-key") + .trackType(trackType) + .build(); + DDIntakeWriter writer = + DDIntakeWriter.builder() + .addTrack(trackType, api) + .healthMetrics(healthMetrics) + .traceBufferSize(bufferSize) + .alwaysFlush(false) + .build(); + writer.start(); + + // gate responses + responseSemaphore.acquire(); + + try { + // sanity check coordination mechanism of test + // release to allow response to be generated + responseSemaphore.release(); + writer.flush(); + + // reacquire semaphore to stall further responses + responseSemaphore.acquire(); + + // when: write a single trace and flush + // with responseSemaphore held, the response is blocked but may still time out + writer.write(minimalTrace); + numWritten += 1; + + assertEquals(0, numFailedPublish.get()); + assertEquals(numWritten, numPublished.get()); + assertEquals(numWritten, numPublished.get() + numFailedPublish.get()); + assertEquals(1, numFlushes.get()); + + // when: send many traces to fill the sender queue... + // loop until outstanding requests > finished requests + while (writer.traceProcessingWorker.getRemainingCapacity() > 0 + || numFailedPublish.get() == 0) { + writer.write(minimalTrace); + numWritten += 1; + } + + assertTrue(numFailedPublish.get() > 0); + assertEquals(numWritten, numPublished.get() + numFailedPublish.get()); + + // with both disruptor & queue full, should reject everything + int expectedRejects = 100; + for (int i = 1; i <= expectedRejects; i++) { + writer.write(minimalTrace); + numWritten += 1; + } + + assertEquals(numWritten, numPublished.get() + numFailedPublish.get()); + } finally { + responseSemaphore.release(); + writer.close(); + intake.close(); + } + } + + @Test + void multiThreaded() throws Exception { + TrackType trackType = TrackType.CITESTCYCLE; + String apiVersion = "v2"; + AtomicInteger numPublished = new AtomicInteger(0); + AtomicInteger numFailedPublish = new AtomicInteger(0); + AtomicInteger numRepSent = new AtomicInteger(0); + + List minimalTrace = createMinimalTrace(); + String path = buildIntakePath(trackType, apiVersion); + + JavaTestHttpServer intake = + JavaTestHttpServer.httpServer( + server -> + server.handlers(h -> h.post(path, api -> api.getResponse().status(200).send()))); + + // This test focuses just on failed publish, so not verifying every callback + HealthMetrics healthMetrics = mock(HealthMetrics.class); + doAnswer( + invocation -> { + numPublished.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onPublish(any(), anyInt()); + doAnswer( + invocation -> { + numFailedPublish.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onFailedPublish(anyInt(), anyInt()); + doAnswer( + invocation -> { + int repCount = invocation.getArgument(0); + numRepSent.addAndGet(repCount); + return null; + }) + .when(healthMetrics) + .onSend(anyInt(), anyInt(), any()); + + HttpUrl hostUrl = HttpUrl.get(intake.getAddress()); + okhttp3.OkHttpClient httpClient = OkHttpUtils.buildHttpClient(hostUrl, 1000); + DDIntakeApi api = + DDIntakeApi.builder() + .hostUrl(hostUrl) + .httpClient(httpClient) + .apiKey("my-api-key") + .trackType(trackType) + .build(); + DDIntakeWriter writer = + DDIntakeWriter.builder() + .addTrack(trackType, api) + .monitoring(monitoring) + .healthMetrics(healthMetrics) + .alwaysFlush(false) + .build(); + writer.start(); + + try { + Runnable producer = + () -> { + for (int i = 1; i <= 100; i++) { + writer.write(minimalTrace); + } + }; + + Thread t1 = new Thread(producer); + t1.start(); + + Thread t2 = new Thread(producer); + t2.start(); + + t1.join(); + t2.join(); + + writer.flush(); + + // then: conditions.eventually { assert numPublished.get() == 200 && numRepSent.get() == 200 } + int totalTraces = 100 + 100; + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + if (numPublished.get() == totalTraces && numRepSent.get() == totalTraces) { + break; + } + Thread.sleep(50); + } + assertEquals(totalTraces, numPublished.get()); + assertEquals(totalTraces, numRepSent.get()); + } finally { + writer.close(); + intake.close(); + } + } + + @Test + void statsdSuccess() { + TrackType trackType = TrackType.CITESTCYCLE; + String apiVersion = "v2"; + AtomicInteger numTracesAccepted = new AtomicInteger(0); + AtomicInteger numRequests = new AtomicInteger(0); + AtomicInteger numResponses = new AtomicInteger(0); + + List minimalTrace = createMinimalTrace(); + String path = buildIntakePath(trackType, apiVersion); + + JavaTestHttpServer intake = + JavaTestHttpServer.httpServer( + server -> + server.handlers(h -> h.post(path, api -> api.getResponse().status(200).send()))); + + HealthMetrics healthMetrics = mock(HealthMetrics.class); + doAnswer( + invocation -> { + numTracesAccepted.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onPublish(any(), anyInt()); + doAnswer( + invocation -> { + numRequests.incrementAndGet(); + numResponses.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onSend(anyInt(), anyInt(), any()); + + HttpUrl hostUrl = HttpUrl.get(intake.getAddress()); + okhttp3.OkHttpClient httpClient = OkHttpUtils.buildHttpClient(hostUrl, 1000); + DDIntakeApi api = + DDIntakeApi.builder() + .hostUrl(hostUrl) + .httpClient(httpClient) + .apiKey("my-api-key") + .trackType(trackType) + .build(); + DDIntakeWriter writer = + DDIntakeWriter.builder() + .addTrack(trackType, api) + .monitoring(monitoring) + .healthMetrics(healthMetrics) + .alwaysFlush(false) + .build(); + writer.start(); + + try { + writer.write(minimalTrace); + writer.flush(); + + assertEquals(1, numTracesAccepted.get()); + assertEquals(1, numRequests.get()); + assertEquals(1, numResponses.get()); + } finally { + intake.close(); + writer.close(); + } + } + + @Test + void statsdCommFailure() throws Exception { + TrackType trackType = TrackType.CITESTCYCLE; + List minimalTrace = createMinimalTrace(); + + DDIntakeApi api = mock(DDIntakeApi.class); + doAnswer(invocation -> RemoteApi.Response.failed(new IOException("comm error"))) + .when(api) + .sendSerializedTraces(any()); + + CountDownLatch latch = new CountDownLatch(2); + StatsDClient statsd = mock(StatsDClient.class); + TracerHealthMetrics healthMetrics = new TracerHealthMetrics(statsd, 100, TimeUnit.MILLISECONDS); + DDIntakeWriter writer = + DDIntakeWriter.builder() + .addTrack(trackType, api) + .monitoring(monitoring) + .healthMetrics(healthMetrics) + .alwaysFlush(false) + .build(); + healthMetrics.start(); + writer.start(); + + // Set up stubs with countDown BEFORE the action + doAnswer( + invocation -> { + latch.countDown(); + return null; + }) + .when(statsd) + .count(anyString(), anyLong()); + + writer.write(minimalTrace); + writer.flush(); + latch.await(10, TimeUnit.SECONDS); + + verify(statsd, times(1)).count("api.requests.total", 1L); + verify(statsd, never()).incrementCounter("api.responses.total"); + verify(statsd, times(1)).count("api.errors.total", 1L); + + writer.close(); + healthMetrics.close(); + } + + static String buildIntakePath(TrackType trackType, String apiVersion) { + return String.format("/api/%s/%s", apiVersion, trackType.name().toLowerCase()); + } + + static int calculateSize(List trace, RemoteMapper mapper) { + AtomicInteger size = new AtomicInteger(); + MsgPackWriter packer = + new MsgPackWriter( + new FlushingBuffer( + mapper.messageBufferSize(), + (messageCount, buffer) -> size.set(buffer.limit() - buffer.position()))); + packer.format(trace, mapper); + packer.flush(); + return size.get(); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterTest.java new file mode 100644 index 00000000000..eb7f00ac75e --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterTest.java @@ -0,0 +1,222 @@ +package datadog.trace.common.writer; + +import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.ENQUEUED_FOR_SERIALIZATION; +import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.ENQUEUED_FOR_SINGLE_SPAN_SAMPLING; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.metrics.api.statsd.StatsDClient; +import datadog.metrics.impl.MonitoringImpl; +import datadog.trace.api.intake.TrackType; +import datadog.trace.common.writer.ddagent.DDAgentApi; +import datadog.trace.common.writer.ddagent.DDAgentMapperDiscovery; +import datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult; +import datadog.trace.core.CoreTracer; +import datadog.trace.core.DDCoreJavaSpecification; +import datadog.trace.core.DDSpan; +import datadog.trace.core.monitor.HealthMetrics; +import datadog.trace.core.propagation.PropagationTags; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.tabletest.junit.TableTest; + +class DDIntakeWriterTest extends DDCoreJavaSpecification { + + HealthMetrics healthMetrics = mock(HealthMetrics.class); + TraceProcessingWorker worker = mock(TraceProcessingWorker.class); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + DDAgentApi api = mock(DDAgentApi.class); + MonitoringImpl monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS); + PayloadDispatcherImpl dispatcher = + new PayloadDispatcherImpl( + new DDAgentMapperDiscovery(discovery), api, healthMetrics, monitoring); + DDIntakeWriter writer = new DDIntakeWriter(worker, dispatcher, healthMetrics, false); + + // Only used to create spans + CoreTracer dummyTracer; + + @BeforeEach + void setup() { + dummyTracer = tracerBuilder().writer(new ListWriter()).build(); + } + + @AfterEach + void cleanup() { + writer.close(); + if (dummyTracer != null) { + dummyTracer.close(); + } + } + + @Test + void testWriterBuilder() { + DDIntakeWriter builtWriter = + DDIntakeWriter.builder().addTrack(TrackType.NOOP, mock(RemoteApi.class)).build(); + + assertNotNull(builtWriter); + } + + @Test + void testWriterStart() { + int capacity = 5; + + when(worker.getCapacity()).thenReturn(capacity); + writer.start(); + + verify(healthMetrics).start(); + verify(worker).start(); + verify(worker).getCapacity(); + verify(healthMetrics).onStart(capacity); + verifyNoMoreInteractions(healthMetrics, worker, discovery, api); + } + + @Test + void testWriterFlush() { + when(worker.flush(1, TimeUnit.SECONDS)).thenReturn(true, false); + + // first flush succeeds + writer.flush(); + + // monitor is notified + verify(worker).flush(1, TimeUnit.SECONDS); + verify(healthMetrics).onFlush(false); + verifyNoMoreInteractions(healthMetrics, worker, discovery, api); + + clearInvocations(healthMetrics, worker, discovery, api); + + // second flush returns false + writer.flush(); + + // no additional monitor notifications + verify(worker).flush(1, TimeUnit.SECONDS); + verifyNoMoreInteractions(healthMetrics, worker, discovery, api); + } + + @Test + void testWriterFlushClosed() { + writer.close(); + clearInvocations(healthMetrics, worker, discovery, api); + + writer.flush(); + + verifyNoMoreInteractions(healthMetrics, worker, discovery, api); + } + + @Test + void testWriterWritePublishSucceeds() { + List trace = + Collections.singletonList( + (DDSpan) dummyTracer.buildSpan("datadog", "fakeOperation").start()); + + // publish succeeds + when(worker.publish(any(), anyInt(), eq(trace))).thenReturn(ENQUEUED_FOR_SERIALIZATION); + when(worker.flush(anyLong(), any(TimeUnit.class))).thenReturn(true); + writer.write(trace); + + // monitor is notified of successful publication + verify(worker).publish(any(), anyInt(), eq(trace)); + verify(healthMetrics).onPublish(any(), anyInt()); + verifyNoMoreInteractions(healthMetrics); + } + + @Test + void testWriterWritePublishForSingleSpanSampling() { + List trace = + Collections.singletonList( + (DDSpan) dummyTracer.buildSpan("datadog", "fakeOperation").start()); + + // publish succeeds for single span sampling + when(worker.publish(any(), anyInt(), eq(trace))).thenReturn(ENQUEUED_FOR_SINGLE_SPAN_SAMPLING); + when(worker.flush(anyLong(), any(TimeUnit.class))).thenReturn(true); + writer.write(trace); + + // monitor should not call onPublish for single span sampling + verify(worker).publish(any(), anyInt(), eq(trace)); + verifyNoMoreInteractions(healthMetrics); + } + + @TableTest({ + "scenario | publishResult ", + "buffer overflow | DROPPED_BUFFER_OVERFLOW", + "dropped by policy | DROPPED_BY_POLICY " + }) + void testWriterWritePublishFails(PublishResult publishResult) { + List trace = + Collections.singletonList( + (DDSpan) dummyTracer.buildSpan("datadog", "fakeOperation").start()); + + // publish fails + when(worker.publish(any(), anyInt(), eq(trace))).thenReturn(publishResult); + when(worker.flush(anyLong(), any(TimeUnit.class))).thenReturn(true); + writer.write(trace); + + // monitor is notified of unsuccessful publication + verify(worker).publish(any(), anyInt(), eq(trace)); + verify(healthMetrics).onFailedPublish(anyInt(), eq(1)); + verifyNoMoreInteractions(healthMetrics); + } + + @Test + void testEmptyTracesShouldBeReportedAsFailures() { + // trace is empty + when(worker.flush(anyLong(), any(TimeUnit.class))).thenReturn(true); + writer.write(Collections.emptyList()); + + // monitor is notified of unsuccessful publication + verify(healthMetrics).onFailedPublish(anyInt(), eq(0)); + verifyNoMoreInteractions(healthMetrics); + } + + @Test + void testWriterWriteClosed() { + writer.close(); + clearInvocations(healthMetrics, worker, discovery, api); + List trace = + Collections.singletonList( + (DDSpan) dummyTracer.buildSpan("datadog", "fakeOperation").start()); + + when(worker.flush(anyLong(), any(TimeUnit.class))).thenReturn(true); + writer.write(trace); + + verify(healthMetrics).onFailedPublish(anyInt(), eq(1)); + verifyNoMoreInteractions(healthMetrics); + } + + @TableTest({ + "scenario | publishResult ", + "dropped by policy | DROPPED_BY_POLICY ", + "buffer overflow | DROPPED_BUFFER_OVERFLOW" + }) + void testDroppedTraceIsCounted(PublishResult publishResult) { + // setup - use local mocks + PayloadDispatcherImpl localDispatcher = mock(PayloadDispatcherImpl.class); + DDIntakeWriter localWriter = new DDIntakeWriter(worker, localDispatcher, healthMetrics, true); + + DDSpan p0 = newSpan(); + List trace = java.util.Arrays.asList(p0, newSpan()); + + when(worker.publish(eq(trace.get(0)), anyInt(), eq(trace))).thenReturn(publishResult); + localWriter.write(trace); + + verify(worker).publish(eq(trace.get(0)), anyInt(), eq(trace)); + verify(localDispatcher).onDroppedTrace(trace.size()); + } + + DDSpan newSpan() { + // Use the UNSET-priority variant so setSamplingPriority() can change the priority later + return buildSpan(0L, "test.tag", "test.value", PropagationTags.factory().empty()); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/MultiWriterTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/MultiWriterTest.java new file mode 100644 index 00000000000..3dd2819f39d --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/MultiWriterTest.java @@ -0,0 +1,79 @@ +package datadog.trace.common.writer; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import datadog.trace.core.DDSpan; +import datadog.trace.test.util.DDJavaSpecification; +import java.util.LinkedList; +import java.util.List; +import org.junit.jupiter.api.Test; + +class MultiWriterTest extends DDJavaSpecification { + + @Test + void testThatMultiWriterDelegatesToAll() { + Writer[] writers = new Writer[3]; + Writer mockW1 = mock(Writer.class); + Writer mockW2 = mock(Writer.class); + writers[0] = mockW1; + // null in position 1 to check that we skip that + writers[2] = mockW2; + MultiWriter writer = new MultiWriter(writers); + List trace = new LinkedList<>(); + + writer.start(); + + verify(mockW1).start(); + verify(mockW2).start(); + verifyNoMoreInteractions(mockW1, mockW2); + clearInvocations(mockW1, mockW2); + + writer.write(trace); + + verify(mockW1).write(trace); + verify(mockW2).write(trace); + verifyNoMoreInteractions(mockW1, mockW2); + clearInvocations(mockW1, mockW2); + + // flush (both return true) + when(mockW1.flush()).thenReturn(true); + when(mockW2.flush()).thenReturn(true); + boolean flushed = writer.flush(); + + verify(mockW1).flush(); + verify(mockW2).flush(); + verifyNoMoreInteractions(mockW1, mockW2); + assertTrue(flushed); + clearInvocations(mockW1, mockW2); + + // flush (one returns false) + when(mockW1.flush()).thenReturn(true); + when(mockW2.flush()).thenReturn(false); + boolean notFlushed = writer.flush(); + + verify(mockW1).flush(); + verify(mockW2).flush(); + verifyNoMoreInteractions(mockW1, mockW2); + assertFalse(notFlushed); + clearInvocations(mockW1, mockW2); + + writer.close(); + + verify(mockW1).close(); + verify(mockW2).close(); + verifyNoMoreInteractions(mockW1, mockW2); + clearInvocations(mockW1, mockW2); + + writer.incrementDropCounts(0); + + verify(mockW1).incrementDropCounts(0); + verify(mockW2).incrementDropCounts(0); + verifyNoMoreInteractions(mockW1, mockW2); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/PayloadDispatcherImplTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/PayloadDispatcherImplTest.java new file mode 100644 index 00000000000..2bf49f69fc3 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/PayloadDispatcherImplTest.java @@ -0,0 +1,228 @@ +package datadog.trace.common.writer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.intThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.metrics.api.statsd.StatsDClient; +import datadog.metrics.impl.MonitoringImpl; +import datadog.trace.api.DDSpanId; +import datadog.trace.api.DDTraceId; +import datadog.trace.api.datastreams.NoopPathwayContext; +import datadog.trace.api.sampling.PrioritySampling; +import datadog.trace.common.writer.ddagent.DDAgentApi; +import datadog.trace.common.writer.ddagent.DDAgentMapperDiscovery; +import datadog.trace.core.CoreTracer; +import datadog.trace.core.DDSpan; +import datadog.trace.core.DDSpanContext; +import datadog.trace.core.PendingTrace; +import datadog.trace.core.monitor.HealthMetrics; +import datadog.trace.core.propagation.PropagationTags; +import datadog.trace.test.util.DDJavaSpecification; +import java.lang.reflect.Constructor; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.tabletest.junit.TableTest; + +class PayloadDispatcherImplTest extends DDJavaSpecification { + + static final MonitoringImpl monitoring = + new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS); + + // Groovy baseline: v0.5 ~5.5s, v0.4 ~1.3s; Java has higher mock overhead so use 30s timeout + @Timeout(30) + @TableTest({"scenario | traceEndpoint", "v0.5 | 'v0.5/traces'", "v0.4 | 'v0.4/traces'"}) + void testFlushAutomaticallyWhenDataLimitIsBreached(String traceEndpoint) throws Exception { + AtomicBoolean flushed = new AtomicBoolean(); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + when(discovery.getTraceEndpoint()).thenReturn(traceEndpoint); + DDAgentApi api = mock(DDAgentApi.class); + when(api.sendSerializedTraces(any())) + .thenAnswer( + inv -> { + flushed.set(true); + return RemoteApi.Response.success(200); + }); + PayloadDispatcherImpl dispatcher = + new PayloadDispatcherImpl( + new DDAgentMapperDiscovery(discovery), api, healthMetrics, monitoring); + List trace = Collections.singletonList(realSpan()); + + while (!flushed.get()) { + dispatcher.addTrace(trace); + } + + // the dispatcher has flushed + assertTrue(flushed.get()); + } + + @TableTest({ + "scenario | traceEndpoint | traceCount", + "v0.4 1 trace | 'v0.4/traces' | 1 ", + "v0.4 10 traces | 'v0.4/traces' | 10 ", + "v0.4 100 traces | 'v0.4/traces' | 100 ", + "v0.5 1 trace | 'v0.5/traces' | 1 ", + "v0.5 10 traces | 'v0.5/traces' | 10 ", + "v0.5 100 traces | 'v0.5/traces' | 100 " + }) + void testShouldFlushBufferOnDemand(String traceEndpoint, int traceCount) throws Exception { + HealthMetrics healthMetrics = mock(HealthMetrics.class); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + DDAgentApi api = mock(DDAgentApi.class); + when(discovery.getTraceEndpoint()).thenReturn(traceEndpoint); + when(api.sendSerializedTraces(any())).thenReturn(RemoteApi.Response.success(200)); + PayloadDispatcherImpl dispatcher = + new PayloadDispatcherImpl( + new DDAgentMapperDiscovery(discovery), api, healthMetrics, monitoring); + List trace = Collections.singletonList(realSpan()); + + for (int i = 0; i < traceCount; ++i) { + dispatcher.addTrace(trace); + } + dispatcher.flush(); + + verify(discovery, org.mockito.Mockito.times(2)).getTraceEndpoint(); + verify(healthMetrics).onSerialize(intThat(size -> size > 0)); + verify(api).sendSerializedTraces(argThat(p -> p.traceCount() == traceCount)); + } + + @TableTest({ + "scenario | traceEndpoint | traceCount", + "v0.4 1 trace | 'v0.4/traces' | 1 ", + "v0.4 10 traces | 'v0.4/traces' | 10 ", + "v0.4 100 traces | 'v0.4/traces' | 100 ", + "v0.5 1 trace | 'v0.5/traces' | 1 ", + "v0.5 10 traces | 'v0.5/traces' | 10 ", + "v0.5 100 traces | 'v0.5/traces' | 100 " + }) + void testShouldReportFailedRequestToMonitor(String traceEndpoint, int traceCount) + throws Exception { + HealthMetrics healthMetrics = mock(HealthMetrics.class); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + DDAgentApi api = mock(DDAgentApi.class); + when(discovery.getTraceEndpoint()).thenReturn(traceEndpoint); + when(api.sendSerializedTraces(any())).thenReturn(RemoteApi.Response.failed(400)); + PayloadDispatcherImpl dispatcher = + new PayloadDispatcherImpl( + new DDAgentMapperDiscovery(discovery), api, healthMetrics, monitoring); + List trace = Collections.singletonList(realSpan()); + + for (int i = 0; i < traceCount; ++i) { + dispatcher.addTrace(trace); + } + dispatcher.flush(); + + verify(discovery, org.mockito.Mockito.times(2)).getTraceEndpoint(); + verify(healthMetrics).onSerialize(intThat(size -> size > 0)); + verify(api).sendSerializedTraces(argThat(p -> p.traceCount() == traceCount)); + } + + @Test + void testShouldDropTraceWhenThereIsNoAgentConnectivity() throws Exception { + HealthMetrics healthMetrics = mock(HealthMetrics.class); + DDAgentApi api = mock(DDAgentApi.class); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + when(discovery.getTraceEndpoint()).thenReturn(null); + PayloadDispatcherImpl dispatcher = + new PayloadDispatcherImpl( + new DDAgentMapperDiscovery(discovery), api, healthMetrics, monitoring); + List trace = Collections.singletonList(realSpan()); + + dispatcher.addTrace(trace); + + verify(healthMetrics).onFailedPublish(eq((int) PrioritySampling.UNSET), anyInt()); + } + + @Test + void testTraceAndSpanCountsAreResetAfterAccess() { + HealthMetrics healthMetrics = mock(HealthMetrics.class); + DDAgentApi api = mock(DDAgentApi.class); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + when(discovery.getTraceEndpoint()).thenReturn("v0.4/traces"); + PayloadDispatcherImpl dispatcher = + new PayloadDispatcherImpl( + new DDAgentMapperDiscovery(discovery), api, healthMetrics, monitoring); + + // add traces and dropped counts + dispatcher.addTrace(Collections.emptyList()); + dispatcher.onDroppedTrace(20); + dispatcher.onDroppedTrace(2); + Payload payload = dispatcher.newPayload(1, ByteBuffer.allocate(0)); + + // dropped counts are accumulated + assertEquals(22, payload.droppedSpans()); + assertEquals(2, payload.droppedTraces()); + + // create another payload + Payload newPayload = dispatcher.newPayload(1, ByteBuffer.allocate(0)); + + // counts are reset after access + assertEquals(0, newPayload.droppedSpans()); + assertEquals(0, newPayload.droppedTraces()); + } + + DDSpan realSpan() throws Exception { + // getTracer() and mapServiceName() are package-private in TraceCollector; use a custom + // Answer to handle them at runtime without compile-time accessibility issues + PendingTrace trace = + mock( + PendingTrace.class, + invocation -> { + Class returnType = invocation.getMethod().getReturnType(); + if (CoreTracer.class.isAssignableFrom(returnType)) { + // Use RETURNS_DEFAULTS so getTagInterceptor() returns null (matching Groovy Stub + // behavior) + return mock(CoreTracer.class); + } + if (returnType == String.class) { + Object[] args = invocation.getArguments(); + // mapServiceName(String) - return the argument unchanged + if (args.length > 0 && args[0] instanceof String) { + return args[0]; + } + return ""; + } + return org.mockito.Mockito.RETURNS_DEFAULTS.answer(invocation); + }); + DDSpanContext context = + new DDSpanContext( + DDTraceId.ONE, + 1L, + DDSpanId.ZERO, + null, + "", + "", + "", + PrioritySampling.UNSET, + "", + Collections.emptyMap(), + false, + "", + 0, + trace, + null, + null, + NoopPathwayContext.INSTANCE, + false, + PropagationTags.factory().empty()); + Constructor ctor = + DDSpan.class.getDeclaredConstructor( + String.class, long.class, DDSpanContext.class, List.class); + ctor.setAccessible(true); + return ctor.newInstance("test", 0L, context, null); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/PrioritizationTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/PrioritizationTest.java new file mode 100644 index 00000000000..016f0da6198 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/PrioritizationTest.java @@ -0,0 +1,307 @@ +package datadog.trace.common.writer; + +import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.DROPPED_BUFFER_OVERFLOW; +import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.ENQUEUED_FOR_SERIALIZATION; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import datadog.trace.api.sampling.PrioritySampling; +import datadog.trace.common.writer.ddagent.FlushEvent; +import datadog.trace.common.writer.ddagent.Prioritization; +import datadog.trace.common.writer.ddagent.PrioritizationStrategy; +import datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult; +import datadog.trace.core.DDSpan; +import datadog.trace.junit.utils.tabletest.PrioritySamplingConverter; +import datadog.trace.test.util.DDJavaSpecification; +import java.util.Collections; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.params.converter.ConvertWith; +import org.tabletest.junit.TableTest; + +class PrioritizationTest extends DDJavaSpecification { + + @SuppressWarnings("unchecked") + @TableTest({ + "scenario | primaryFull | priority | primaryOffers | secondaryOffers", + "unset full | true | PrioritySampling.UNSET | 2 | 0 ", + "drop full | true | PrioritySampling.SAMPLER_DROP | 0 | 1 ", + "keep full | true | PrioritySampling.SAMPLER_KEEP | 2 | 0 ", + "drop full 2 | true | PrioritySampling.SAMPLER_DROP | 0 | 1 ", + "user keep full | true | PrioritySampling.USER_KEEP | 2 | 0 ", + "unset not full | false | PrioritySampling.UNSET | 1 | 0 ", + "drop not full | false | PrioritySampling.SAMPLER_DROP | 0 | 1 ", + "keep not full | false | PrioritySampling.SAMPLER_KEEP | 1 | 0 ", + "drop not full 2 | false | PrioritySampling.SAMPLER_DROP | 0 | 1 ", + "user keep not full | false | PrioritySampling.USER_KEEP | 1 | 0 " + }) + void testEnsureTraceStrategyTriesToSendKeptAndUnsetPriorityTracesToPrimaryQueue( + boolean primaryFull, + @ConvertWith(PrioritySamplingConverter.class) int priority, + int primaryOffers, + int secondaryOffers) { + List trace = Collections.emptyList(); + Queue primary = mock(Queue.class); + Queue secondary = mock(Queue.class); + PrioritizationStrategy blocking = + Prioritization.ENSURE_TRACE.create(primary, secondary, null, () -> false); + // stub: first offer returns !primaryFull, second offer returns true + when(primary.offer(trace)).thenReturn(!primaryFull, true); + when(secondary.offer(trace)).thenReturn(true); + + PublishResult publishResult = blocking.publish(mock(DDSpan.class), priority, trace); + + assertEquals(ENQUEUED_FOR_SERIALIZATION, publishResult); + verify(primary, times(primaryOffers)).offer(trace); + verify(secondary, times(secondaryOffers)).offer(trace); + } + + @SuppressWarnings("unchecked") + @TableTest({ + "scenario | priority | primaryOffers | secondaryOffers", + "unset | PrioritySampling.UNSET | 1 | 0 ", + "drop | PrioritySampling.SAMPLER_DROP | 0 | 1 ", + "keep | PrioritySampling.SAMPLER_KEEP | 1 | 0 ", + "drop 2 | PrioritySampling.SAMPLER_DROP | 0 | 1 ", + "user keep | PrioritySampling.USER_KEEP | 1 | 0 " + }) + void testFastLaneStrategySendsKeptAndUnsetPriorityTracesToPrimaryQueue( + @ConvertWith(PrioritySamplingConverter.class) int priority, + int primaryOffers, + int secondaryOffers) { + List trace = Collections.emptyList(); + Queue primary = mock(Queue.class); + Queue secondary = mock(Queue.class); + PrioritizationStrategy fastLane = + Prioritization.FAST_LANE.create(primary, secondary, null, () -> false); + + PublishResult publishResult = fastLane.publish(mock(DDSpan.class), priority, trace); + + assertEquals(DROPPED_BUFFER_OVERFLOW, publishResult); + verify(primary, times(primaryOffers)).offer(trace); + verify(secondary, times(secondaryOffers)).offer(trace); + } + + @SuppressWarnings("unchecked") + @TableTest({ + "scenario | priority | primaryOffers | expectedResult ", + "unset | PrioritySampling.UNSET | 1 | ENQUEUED_FOR_SERIALIZATION", + "drop | PrioritySampling.SAMPLER_DROP | 0 | DROPPED_BY_POLICY ", + "keep | PrioritySampling.SAMPLER_KEEP | 1 | ENQUEUED_FOR_SERIALIZATION", + "drop 2 | PrioritySampling.SAMPLER_DROP | 0 | DROPPED_BY_POLICY ", + "user keep | PrioritySampling.USER_KEEP | 1 | ENQUEUED_FOR_SERIALIZATION" + }) + void testFastLaneWithActiveDroppingPolicySendsKeptAndUnsetTracesToPrimaryQueue( + @ConvertWith(PrioritySamplingConverter.class) int priority, + int primaryOffers, + PublishResult expectedResult) { + List trace = Collections.emptyList(); + Queue primary = mock(Queue.class); + Queue secondary = mock(Queue.class); + PrioritizationStrategy drop = + Prioritization.FAST_LANE.create(primary, secondary, null, () -> true); + when(primary.offer(trace)).thenReturn(true); + + PublishResult publishResult = drop.publish(mock(DDSpan.class), priority, trace); + + assertEquals(expectedResult, publishResult); + verify(primary, times(primaryOffers)).offer(trace); + verify(secondary, never()).offer(trace); + } + + @SuppressWarnings("unchecked") + @TableTest({ + "scenario | strategy ", + "fast lane | FAST_LANE ", + "ensure trace | ENSURE_TRACE" + }) + void testStrategyFlushesPrimaryQueue(Prioritization strategy) { + Queue primary = mock(Queue.class); + Queue secondary = mock(Queue.class); + PrioritizationStrategy prioritizationStrategy = + strategy.create(primary, secondary, null, () -> false); + when(primary.offer(any())).thenReturn(true); + + prioritizationStrategy.flush(100, TimeUnit.MILLISECONDS); + + verify(primary).offer(any(FlushEvent.class)); + verify(secondary, never()).offer(any()); + } + + @SuppressWarnings("unchecked") + @TableTest({ + "scenario | strategy | forceKeep | expectedResult ", + "force keep true fast lane | FAST_LANE | true | ENQUEUED_FOR_SERIALIZATION", + "force keep false fast lane | FAST_LANE | false | DROPPED_BY_POLICY " + }) + void testDropStrategyRespectsForceKeep( + Prioritization strategy, boolean forceKeep, PublishResult expectedResult) { + Queue primary = mock(Queue.class); + PrioritizationStrategy drop = strategy.create(primary, null, null, () -> true); + DDSpan root = mock(DDSpan.class); + List trace = Collections.singletonList(root); + when(root.isForceKeep()).thenReturn(forceKeep); + when(primary.offer(trace)).thenReturn(true); + + PublishResult publishResult = drop.publish(root, PrioritySampling.SAMPLER_DROP, trace); + + assertEquals(expectedResult, publishResult); + verify(root).isForceKeep(); + verify(primary, times(forceKeep ? 1 : 0)).offer(trace); + verifyNoMoreInteractions(root, primary); + } + + @SuppressWarnings("unchecked") + @TableTest({ + "scenario | primaryFull | priority | primaryOffers | singleSpanOffers | singleSpanFull | expectedResult ", + "unset full ss-not-full | true | PrioritySampling.UNSET | 2 | 0 | false | ENQUEUED_FOR_SERIALIZATION ", + "drop full ss-not-full | true | PrioritySampling.SAMPLER_DROP | 0 | 1 | false | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING", + "keep full ss-not-full | true | PrioritySampling.SAMPLER_KEEP | 2 | 0 | false | ENQUEUED_FOR_SERIALIZATION ", + "drop full 2 ss-not-full | true | PrioritySampling.SAMPLER_DROP | 0 | 1 | false | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING", + "ukeep full ss-not-full | true | PrioritySampling.USER_KEEP | 2 | 0 | false | ENQUEUED_FOR_SERIALIZATION ", + "unset nfull ss-not-full | false | PrioritySampling.UNSET | 1 | 0 | false | ENQUEUED_FOR_SERIALIZATION ", + "drop nfull ss-not-full | false | PrioritySampling.SAMPLER_DROP | 0 | 1 | false | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING", + "keep nfull ss-not-full | false | PrioritySampling.SAMPLER_KEEP | 1 | 0 | false | ENQUEUED_FOR_SERIALIZATION ", + "drop nfull 2 ss-not-full | false | PrioritySampling.SAMPLER_DROP | 0 | 1 | false | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING", + "ukeep nfull ss-not-full | false | PrioritySampling.USER_KEEP | 1 | 0 | false | ENQUEUED_FOR_SERIALIZATION ", + "unset full ss-full | true | PrioritySampling.UNSET | 2 | 0 | true | ENQUEUED_FOR_SERIALIZATION ", + "drop full ss-full | true | PrioritySampling.SAMPLER_DROP | 0 | 1 | true | DROPPED_BUFFER_OVERFLOW ", + "keep full ss-full | true | PrioritySampling.SAMPLER_KEEP | 2 | 0 | true | ENQUEUED_FOR_SERIALIZATION ", + "drop full 2 ss-full | true | PrioritySampling.SAMPLER_DROP | 0 | 1 | true | DROPPED_BUFFER_OVERFLOW ", + "ukeep full ss-full | true | PrioritySampling.USER_KEEP | 2 | 0 | true | ENQUEUED_FOR_SERIALIZATION ", + "unset nfull ss-full | false | PrioritySampling.UNSET | 1 | 0 | true | ENQUEUED_FOR_SERIALIZATION ", + "drop nfull ss-full | false | PrioritySampling.SAMPLER_DROP | 0 | 1 | true | DROPPED_BUFFER_OVERFLOW ", + "keep nfull ss-full | false | PrioritySampling.SAMPLER_KEEP | 1 | 0 | true | ENQUEUED_FOR_SERIALIZATION ", + "drop nfull 2 ss-full | false | PrioritySampling.SAMPLER_DROP | 0 | 1 | true | DROPPED_BUFFER_OVERFLOW ", + "ukeep nfull ss-full | false | PrioritySampling.USER_KEEP | 1 | 0 | true | ENQUEUED_FOR_SERIALIZATION " + }) + void testEnsureTraceStrategyWithSpanSamplingQueue( + boolean primaryFull, + @ConvertWith(PrioritySamplingConverter.class) int priority, + int primaryOffers, + int singleSpanOffers, + boolean singleSpanFull, + PublishResult expectedResult) { + List trace = Collections.emptyList(); + Queue primary = mock(Queue.class); + Queue secondary = mock(Queue.class); + Queue spanSampling = mock(Queue.class); + PrioritizationStrategy blocking = + Prioritization.ENSURE_TRACE.create(primary, secondary, spanSampling, () -> false); + when(primary.offer(trace)).thenReturn(!primaryFull, true); + when(spanSampling.offer(trace)).thenReturn(!singleSpanFull); + + PublishResult publishResult = blocking.publish(mock(DDSpan.class), priority, trace); + + assertEquals(expectedResult, publishResult); + verify(primary, times(primaryOffers)).offer(trace); + verify(secondary, never()).offer(trace); // expect no traces sent to the secondary queue + verify(spanSampling, times(singleSpanOffers)).offer(trace); + } + + @SuppressWarnings("unchecked") + @TableTest({ + "scenario | priority | primaryOffers | singleSpanOffers | singleSpanFull | expectedResult ", + "unset ss-not-full | PrioritySampling.UNSET | 1 | 0 | false | ENQUEUED_FOR_SERIALIZATION ", + "drop ss-not-full | PrioritySampling.SAMPLER_DROP | 0 | 1 | false | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING", + "keep ss-not-full | PrioritySampling.SAMPLER_KEEP | 1 | 0 | false | ENQUEUED_FOR_SERIALIZATION ", + "drop 2 ss-not-full | PrioritySampling.SAMPLER_DROP | 0 | 1 | false | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING", + "user keep ss-not-full | PrioritySampling.USER_KEEP | 1 | 0 | false | ENQUEUED_FOR_SERIALIZATION ", + "unset ss-full | PrioritySampling.UNSET | 1 | 0 | true | ENQUEUED_FOR_SERIALIZATION ", + "drop ss-full | PrioritySampling.SAMPLER_DROP | 0 | 1 | true | DROPPED_BUFFER_OVERFLOW ", + "keep ss-full | PrioritySampling.SAMPLER_KEEP | 1 | 0 | true | ENQUEUED_FOR_SERIALIZATION ", + "drop 2 ss-full | PrioritySampling.SAMPLER_DROP | 0 | 1 | true | DROPPED_BUFFER_OVERFLOW ", + "user keep ss-full | PrioritySampling.USER_KEEP | 1 | 0 | true | ENQUEUED_FOR_SERIALIZATION " + }) + void testFastLaneStrategyWithSpanSamplingQueue( + @ConvertWith(PrioritySamplingConverter.class) int priority, + int primaryOffers, + int singleSpanOffers, + boolean singleSpanFull, + PublishResult expectedResult) { + List trace = Collections.emptyList(); + Queue primary = mock(Queue.class); + Queue secondary = mock(Queue.class); + Queue spanSampling = mock(Queue.class); + PrioritizationStrategy fastLane = + Prioritization.FAST_LANE.create(primary, secondary, spanSampling, () -> false); + when(primary.offer(trace)).thenReturn(true); + when(spanSampling.offer(trace)).thenReturn(!singleSpanFull); + + PublishResult publishResult = fastLane.publish(mock(DDSpan.class), priority, trace); + + assertEquals(expectedResult, publishResult); + verify(primary, times(primaryOffers)).offer(trace); + verify(secondary, never()).offer(any()); // expect no traces sent to the secondary queue + verify(spanSampling, times(singleSpanOffers)).offer(trace); + } + + @SuppressWarnings("unchecked") + @TableTest({ + "scenario | priority | primaryOffers | singleSpanOffers | expectedResult ", + "unset | PrioritySampling.UNSET | 1 | 0 | ENQUEUED_FOR_SERIALIZATION ", + "drop | PrioritySampling.SAMPLER_DROP | 0 | 1 | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING", + "keep | PrioritySampling.SAMPLER_KEEP | 1 | 0 | ENQUEUED_FOR_SERIALIZATION ", + "drop 2 | PrioritySampling.SAMPLER_DROP | 0 | 1 | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING", + "user keep | PrioritySampling.USER_KEEP | 1 | 0 | ENQUEUED_FOR_SERIALIZATION " + }) + void testFastLaneWithActiveDroppingPolicySendToSingleSpanSampling( + @ConvertWith(PrioritySamplingConverter.class) int priority, + int primaryOffers, + int singleSpanOffers, + PublishResult expectedResult) { + List trace = Collections.emptyList(); + Queue primary = mock(Queue.class); + Queue secondary = mock(Queue.class); + Queue spanSampling = mock(Queue.class); + PrioritizationStrategy drop = + Prioritization.FAST_LANE.create(primary, secondary, spanSampling, () -> true); + when(primary.offer(trace)).thenReturn(true); + when(spanSampling.offer(trace)).thenReturn(true); + + PublishResult publishResult = drop.publish(mock(DDSpan.class), priority, trace); + + assertEquals(expectedResult, publishResult); + verify(primary, times(primaryOffers)).offer(trace); + verify(secondary, never()).offer(trace); + verify(spanSampling, times(singleSpanOffers)).offer(trace); + } + + @SuppressWarnings("unchecked") + @TableTest({ + "scenario | strategy | forceKeep | singleSpanFull | expectedResult ", + "force keep true full | FAST_LANE | true | true | ENQUEUED_FOR_SERIALIZATION ", + "force keep false full | FAST_LANE | false | true | DROPPED_BUFFER_OVERFLOW ", + "force keep true not full | FAST_LANE | true | false | ENQUEUED_FOR_SERIALIZATION ", + "force keep false not full | FAST_LANE | false | false | ENQUEUED_FOR_SINGLE_SPAN_SAMPLING" + }) + void testSpanSamplingDropStrategyRespectsForceKeep( + Prioritization strategy, + boolean forceKeep, + boolean singleSpanFull, + PublishResult expectedResult) { + Queue primary = mock(Queue.class); + Queue spanSampling = mock(Queue.class); + PrioritizationStrategy drop = strategy.create(primary, null, spanSampling, () -> true); + DDSpan root = mock(DDSpan.class); + List trace = Collections.singletonList(root); + when(root.isForceKeep()).thenReturn(forceKeep); + when(primary.offer(trace)).thenReturn(true); + when(spanSampling.offer(trace)).thenReturn(!singleSpanFull); + + PublishResult publishResult = drop.publish(root, PrioritySampling.SAMPLER_DROP, trace); + + assertEquals(expectedResult, publishResult); + verify(root).isForceKeep(); + verify(primary, times(forceKeep ? 1 : 0)).offer(trace); + verify(spanSampling, times(forceKeep ? 0 : 1)).offer(trace); + verifyNoMoreInteractions(root, primary, spanSampling); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/SerializationTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/SerializationTest.java new file mode 100644 index 00000000000..a0254db18d0 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/SerializationTest.java @@ -0,0 +1,65 @@ +package datadog.trace.common.writer; + +import static java.util.Collections.singletonMap; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import datadog.trace.test.util.DDJavaSpecification; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.msgpack.core.MessageBufferPacker; +import org.msgpack.core.MessagePack; +import org.msgpack.jackson.dataformat.MessagePackFactory; + +class SerializationTest extends DDJavaSpecification { + + @Test + void testJsonMapperSerialization() throws Exception { + // setup + ObjectMapper mapper = new ObjectMapper(); + Map map = singletonMap("key1", "val1"); + byte[] serializedMap = mapper.writeValueAsBytes(map); + byte[] serializedList = ("[" + new String(serializedMap) + "]").getBytes(); + + // when + List> result = + mapper.readValue(serializedList, new TypeReference>>() {}); + + // then + assertEquals(Collections.singletonList(map), result); + assertEquals("[{\"key1\":\"val1\"}]", new String(serializedList)); + } + + @Test + void testMsgpackMapperSerialization() throws Exception { + // setup + ObjectMapper mapper = new ObjectMapper(new MessagePackFactory()); + // GStrings get odd results in the serializer. + List> input = new ArrayList<>(); + for (int i = 1; i <= 1; i++) { + input.add(singletonMap("key" + i, "val" + i)); + } + List serializedMaps = new ArrayList<>(); + for (Map item : input) { + serializedMaps.add(mapper.writeValueAsBytes(item)); + } + + MessageBufferPacker packer = MessagePack.newDefaultBufferPacker(); + packer.packArrayHeader(serializedMaps.size()); + for (byte[] bytes : serializedMaps) { + packer.writePayload(bytes); + } + byte[] serializedList = packer.toByteArray(); + + // when + List> result = + mapper.readValue(serializedList, new TypeReference>>() {}); + + // then + assertEquals(input, result); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/SpanSamplingWorkerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/SpanSamplingWorkerTest.java new file mode 100644 index 00000000000..f03ecc985df --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/SpanSamplingWorkerTest.java @@ -0,0 +1,373 @@ +package datadog.trace.common.writer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.trace.api.sampling.PrioritySampling; +import datadog.trace.common.sampling.SingleSpanSampler; +import datadog.trace.core.DDSpan; +import datadog.trace.core.monitor.HealthMetrics; +import datadog.trace.test.util.DDJavaSpecification; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import org.tabletest.junit.TableTest; + +class SpanSamplingWorkerTest extends DDJavaSpecification { + + @Test + void testSendOnlySampledSpansToTheSampledSpanQueue() throws InterruptedException { + BlockingQueue primaryQueue = new LinkedBlockingDeque<>(10); + BlockingQueue secondaryQueue = new LinkedBlockingDeque<>(10); + SingleSpanSampler singleSpanSampler = mock(SingleSpanSampler.class); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + SpanSamplingWorker worker = + SpanSamplingWorker.build( + 10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, () -> false); + worker.start(); + DDSpan span1 = mock(DDSpan.class); + DDSpan span2 = mock(DDSpan.class); + DDSpan span3 = mock(DDSpan.class); + when(singleSpanSampler.setSamplingPriority(span1)).thenReturn(true); + when(singleSpanSampler.setSamplingPriority(span2)).thenReturn(false); + when(singleSpanSampler.setSamplingPriority(span3)).thenReturn(true); + + worker.getSpanSamplingQueue().offer(Arrays.asList(span1, span2, span3)); + + assertEquals(Arrays.asList(span1, span3), primaryQueue.take()); + assertEquals(Arrays.asList(span2), secondaryQueue.take()); + + worker.close(); + } + + @Test + void testHandleMultipleTraces() throws InterruptedException { + BlockingQueue primaryQueue = new LinkedBlockingDeque<>(10); + BlockingQueue secondaryQueue = new LinkedBlockingDeque<>(10); + SingleSpanSampler singleSpanSampler = mock(SingleSpanSampler.class); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + SpanSamplingWorker worker = + SpanSamplingWorker.build( + 10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, () -> false); + worker.start(); + DDSpan span1 = mock(DDSpan.class); + DDSpan span2 = mock(DDSpan.class); + DDSpan span3 = mock(DDSpan.class); + when(singleSpanSampler.setSamplingPriority(span1)).thenReturn(true); + when(singleSpanSampler.setSamplingPriority(span2)).thenReturn(false); + when(singleSpanSampler.setSamplingPriority(span3)).thenReturn(true); + DDSpan span4 = mock(DDSpan.class); + DDSpan span5 = mock(DDSpan.class); + when(singleSpanSampler.setSamplingPriority(span4)).thenReturn(true); + when(singleSpanSampler.setSamplingPriority(span5)).thenReturn(false); + + worker.getSpanSamplingQueue().offer(Arrays.asList(span1, span2, span3)); + worker.getSpanSamplingQueue().offer(Arrays.asList(span4, span5)); + + assertEquals(Arrays.asList(span1, span3), primaryQueue.take()); + assertEquals(Arrays.asList(span2), secondaryQueue.take()); + assertEquals(Arrays.asList(span4), primaryQueue.take()); + assertEquals(Arrays.asList(span5), secondaryQueue.take()); + + worker.close(); + } + + @Test + void testSkipTracesWithNoSampledSpans() throws InterruptedException { + BlockingQueue primaryQueue = new LinkedBlockingDeque<>(10); + BlockingQueue secondaryQueue = new LinkedBlockingDeque<>(10); + SingleSpanSampler singleSpanSampler = mock(SingleSpanSampler.class); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + SpanSamplingWorker worker = + SpanSamplingWorker.build( + 10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, () -> false); + worker.start(); + DDSpan span1 = mock(DDSpan.class); + DDSpan span2 = mock(DDSpan.class); + DDSpan span3 = mock(DDSpan.class); + when(singleSpanSampler.setSamplingPriority(span1)).thenReturn(true); + when(singleSpanSampler.setSamplingPriority(span2)).thenReturn(false); + when(singleSpanSampler.setSamplingPriority(span3)).thenReturn(true); + DDSpan span4 = mock(DDSpan.class); + DDSpan span5 = mock(DDSpan.class); + when(singleSpanSampler.setSamplingPriority(span4)).thenReturn(false); + when(singleSpanSampler.setSamplingPriority(span5)).thenReturn(false); + DDSpan span6 = mock(DDSpan.class); + DDSpan span7 = mock(DDSpan.class); + when(singleSpanSampler.setSamplingPriority(span6)).thenReturn(true); + when(singleSpanSampler.setSamplingPriority(span7)).thenReturn(true); + + assertTrue(worker.getSpanSamplingQueue().offer(Arrays.asList(span1, span2, span3))); + assertTrue(worker.getSpanSamplingQueue().offer(Arrays.asList(span4, span5))); + assertTrue(worker.getSpanSamplingQueue().offer(Arrays.asList(span6, span7))); + + assertEquals(Arrays.asList(span1, span3), primaryQueue.take()); + assertEquals(Arrays.asList(span2), secondaryQueue.take()); + assertEquals(Arrays.asList(span4, span5), secondaryQueue.take()); + assertEquals(Arrays.asList(span6, span7), primaryQueue.take()); + + worker.close(); + } + + @Test + void testIgnoreEmptyTraces() throws InterruptedException { + BlockingQueue primaryQueue = new LinkedBlockingDeque<>(10); + BlockingQueue secondaryQueue = new LinkedBlockingDeque<>(10); + SingleSpanSampler singleSpanSampler = mock(SingleSpanSampler.class); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + SpanSamplingWorker worker = + SpanSamplingWorker.build( + 10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, () -> false); + worker.start(); + DDSpan span1 = mock(DDSpan.class); + when(singleSpanSampler.setSamplingPriority(span1)).thenReturn(true); + + assertTrue(worker.getSpanSamplingQueue().offer(java.util.Collections.emptyList())); + assertTrue(worker.getSpanSamplingQueue().offer(Arrays.asList(span1))); + + assertEquals(Arrays.asList(span1), primaryQueue.take()); + assertTrue(secondaryQueue.isEmpty()); + + worker.close(); + } + + @Test + void testUpdateDroppedTracesMetricWhenNoTracerSpansHaveBeenSampled() throws InterruptedException { + BlockingQueue primaryQueue = new LinkedBlockingDeque<>(10); + BlockingQueue secondaryQueue = new LinkedBlockingDeque<>(10); + SingleSpanSampler singleSpanSampler = mock(SingleSpanSampler.class); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + CountDownLatch latch = new CountDownLatch(1); + SpanSamplingWorker worker = + new SpanSamplingWorker.DefaultSpanSamplingWorker( + 10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, () -> false) { + @Override + protected void afterOnEvent() { + latch.countDown(); + } + }; + worker.start(); + DDSpan span1 = mock(DDSpan.class); + DDSpan span2 = mock(DDSpan.class); + when(span1.samplingPriority()).thenReturn((int) PrioritySampling.USER_DROP); + when(singleSpanSampler.setSamplingPriority(span1)).thenReturn(false); + when(singleSpanSampler.setSamplingPriority(span2)).thenReturn(false); + List trace = Arrays.asList(span1, span2); + + assertTrue(worker.getSpanSamplingQueue().offer(trace)); + + // wait for processing + assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(primaryQueue.isEmpty()); + assertEquals(trace, secondaryQueue.take()); + verify(healthMetrics).onPublish(trace, PrioritySampling.USER_DROP); + verify(healthMetrics, never()).onFailedPublish(anyInt(), anyInt()); + verify(healthMetrics, never()).onPartialPublish(anyInt()); + + worker.close(); + } + + @Test + void testUpdateDroppedTracesMetricWhenPrimaryQueueIsFull() throws InterruptedException { + BlockingQueue primaryQueue = new LinkedBlockingDeque<>(1); + BlockingQueue secondaryQueue = new LinkedBlockingDeque<>(10); + primaryQueue.offer(java.util.Collections.emptyList()); // occupy the entire queue + SingleSpanSampler singleSpanSampler = mock(SingleSpanSampler.class); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + CountDownLatch latch = new CountDownLatch(1); + SpanSamplingWorker worker = + new SpanSamplingWorker.DefaultSpanSamplingWorker( + 10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, () -> false) { + @Override + protected void afterOnEvent() { + latch.countDown(); + } + }; + worker.start(); + DDSpan span1 = mock(DDSpan.class); + DDSpan span2 = mock(DDSpan.class); + when(span1.samplingPriority()).thenReturn((int) PrioritySampling.SAMPLER_DROP); + when(singleSpanSampler.setSamplingPriority(span1)).thenReturn(false); + when(singleSpanSampler.setSamplingPriority(span2)).thenReturn(true); + List trace = Arrays.asList(span1, span2); + + assertTrue(worker.getSpanSamplingQueue().offer(trace)); + + // wait for processing + assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(secondaryQueue.isEmpty()); + verify(healthMetrics).onFailedPublish(eq((int) PrioritySampling.SAMPLER_DROP), anyInt()); + verify(healthMetrics, never()).onPublish(any(), anyInt()); + verify(healthMetrics, never()).onPartialPublish(anyInt()); + + worker.close(); + } + + @Test + void testUpdatePublishedTracesMetricWhenAllTraceSpansHaveBeenSampled() + throws InterruptedException { + BlockingQueue primaryQueue = new LinkedBlockingDeque<>(10); + BlockingQueue secondaryQueue = new LinkedBlockingDeque<>(10); + SingleSpanSampler singleSpanSampler = mock(SingleSpanSampler.class); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + SpanSamplingWorker worker = + SpanSamplingWorker.build( + 10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, () -> false); + worker.start(); + DDSpan span1 = mock(DDSpan.class); + DDSpan span2 = mock(DDSpan.class); + when(span1.samplingPriority()).thenReturn((int) PrioritySampling.SAMPLER_DROP); + when(singleSpanSampler.setSamplingPriority(span1)).thenReturn(true); + when(singleSpanSampler.setSamplingPriority(span2)).thenReturn(true); + List trace = Arrays.asList(span1, span2); + + assertTrue(worker.getSpanSamplingQueue().offer(trace)); + + // take() blocks until worker has put spans in primaryQueue + assertEquals(trace, primaryQueue.take()); + assertTrue(secondaryQueue.isEmpty()); + // use timeout to wait for healthMetrics to be called after queue operations + verify(healthMetrics, timeout(5000)).onPublish(trace, PrioritySampling.SAMPLER_DROP); + verify(healthMetrics, never()).onFailedPublish(anyInt(), anyInt()); + verify(healthMetrics, never()).onPartialPublish(anyInt()); + + worker.close(); + } + + @Test + void testUpdatePartialTracesMetricWhenSomeSpansDroppedAndSentToSecondaryQueue() + throws InterruptedException { + BlockingQueue primaryQueue = new LinkedBlockingDeque<>(10); + BlockingQueue secondaryQueue = new LinkedBlockingDeque<>(10); + SingleSpanSampler singleSpanSampler = mock(SingleSpanSampler.class); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + SpanSamplingWorker worker = + SpanSamplingWorker.build( + 10, primaryQueue, secondaryQueue, singleSpanSampler, healthMetrics, () -> false); + worker.start(); + DDSpan span1 = mock(DDSpan.class); + DDSpan span2 = mock(DDSpan.class); + DDSpan span3 = mock(DDSpan.class); + when(singleSpanSampler.setSamplingPriority(span1)).thenReturn(false); + when(singleSpanSampler.setSamplingPriority(span2)).thenReturn(true); + when(singleSpanSampler.setSamplingPriority(span3)).thenReturn(false); + List trace = Arrays.asList(span1, span2, span3); + + assertTrue(worker.getSpanSamplingQueue().offer(trace)); + + // take() blocks until worker has put spans in queues + assertEquals(Arrays.asList(span2), primaryQueue.take()); + assertEquals(Arrays.asList(span1, span3), secondaryQueue.take()); + // use timeout to wait for healthMetrics to be called after queue operations + verify(healthMetrics, timeout(5000)).onPublish(trace, PrioritySampling.SAMPLER_DROP); + verify(healthMetrics, never()).onPartialPublish(anyInt()); + verify(healthMetrics, never()).onFailedPublish(anyInt(), anyInt()); + + worker.close(); + } + + @TableTest({ + "scenario | droppingPolicy | secondaryQueueIsFull", + "dropping active | true | false ", + "secondary queue full | false | true ", + "dropping active+full queue | true | true " + }) + void testUpdatePartialTracesMetricWhenSpansDroppedAndSecondaryQueueFullOrDroppingPolicyActive( + boolean droppingPolicy, boolean secondaryQueueIsFull) throws InterruptedException { + BlockingQueue primaryQueue = new LinkedBlockingDeque<>(10); + BlockingQueue secondaryQueue = new LinkedBlockingDeque<>(secondaryQueueIsFull ? 1 : 10); + if (secondaryQueueIsFull) { + // occupy the entire queue + secondaryQueue.offer(java.util.Collections.emptyList()); + } + SingleSpanSampler singleSpanSampler = mock(SingleSpanSampler.class); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + SpanSamplingWorker worker = + new SpanSamplingWorker.DefaultSpanSamplingWorker( + 10, + primaryQueue, + secondaryQueue, + singleSpanSampler, + healthMetrics, + () -> droppingPolicy); + worker.start(); + DDSpan span1 = mock(DDSpan.class); + DDSpan span2 = mock(DDSpan.class); + DDSpan span3 = mock(DDSpan.class); + when(span1.samplingPriority()).thenReturn((int) PrioritySampling.SAMPLER_DROP); + when(singleSpanSampler.setSamplingPriority(span1)).thenReturn(false); + when(singleSpanSampler.setSamplingPriority(span2)).thenReturn(true); + when(singleSpanSampler.setSamplingPriority(span3)).thenReturn(false); + + assertTrue(worker.getSpanSamplingQueue().offer(Arrays.asList(span1, span2, span3))); + + // take() blocks until worker has put span2 in primaryQueue + assertEquals(Arrays.asList(span2), primaryQueue.take()); + // use timeout to wait for healthMetrics to be called after queue operations + verify(healthMetrics, timeout(5000)).onPartialPublish(2); + verify(healthMetrics, never()).onFailedPublish(anyInt(), anyInt()); + verify(healthMetrics, never()).onPublish(any(), anyInt()); + + worker.close(); + } + + @TableTest({ + "scenario | droppingPolicy | secondaryQueueIsFull", + "dropping active | true | false ", + "secondary queue full | false | true ", + "dropping active+full queue | true | true " + }) + void testUpdateFailedPublishMetricWhenAllSpansDroppedAndSecondaryQueueFullOrDroppingPolicyActive( + boolean droppingPolicy, boolean secondaryQueueIsFull) throws InterruptedException { + BlockingQueue primaryQueue = new LinkedBlockingDeque<>(10); + BlockingQueue secondaryQueue = new LinkedBlockingDeque<>(secondaryQueueIsFull ? 1 : 10); + if (secondaryQueueIsFull) { + // occupy the entire queue + secondaryQueue.offer(java.util.Collections.emptyList()); + } + SingleSpanSampler singleSpanSampler = mock(SingleSpanSampler.class); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + CountDownLatch latch = new CountDownLatch(1); + SpanSamplingWorker worker = + new SpanSamplingWorker.DefaultSpanSamplingWorker( + 10, + primaryQueue, + secondaryQueue, + singleSpanSampler, + healthMetrics, + () -> droppingPolicy) { + @Override + protected void afterOnEvent() { + latch.countDown(); + } + }; + worker.start(); + DDSpan span1 = mock(DDSpan.class); + DDSpan span2 = mock(DDSpan.class); + when(span1.samplingPriority()).thenReturn((int) PrioritySampling.SAMPLER_DROP); + when(singleSpanSampler.setSamplingPriority(span1)).thenReturn(false); + when(singleSpanSampler.setSamplingPriority(span2)).thenReturn(false); + + assertTrue(worker.getSpanSamplingQueue().offer(Arrays.asList(span1, span2))); + + // wait for processing via latch + assertTrue(latch.await(10, TimeUnit.SECONDS)); + verify(healthMetrics).onFailedPublish(eq((int) PrioritySampling.SAMPLER_DROP), anyInt()); + verify(healthMetrics, never()).onPartialPublish(anyInt()); + verify(healthMetrics, never()).onPublish(any(), anyInt()); + + worker.close(); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceMapperTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceMapperTest.java new file mode 100644 index 00000000000..2d28a395412 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceMapperTest.java @@ -0,0 +1,124 @@ +package datadog.trace.common.writer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.communication.serialization.ByteBufferConsumer; +import datadog.communication.serialization.FlushingBuffer; +import datadog.communication.serialization.GrowableBuffer; +import datadog.communication.serialization.msgpack.MsgPackWriter; +import datadog.trace.common.writer.ddagent.TraceMapperTestBridge; +import datadog.trace.common.writer.ddagent.TraceMapperV0_5; +import datadog.trace.core.CoreTracer; +import datadog.trace.core.DDCoreJavaSpecification; +import datadog.trace.core.DDSpan; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessageUnpacker; + +class TraceMapperTest extends DDCoreJavaSpecification { + + @Test + void testTraceMapperV05() throws Exception { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + DDSpan span = + (DDSpan) + tracer + .buildSpan("datadog", null) + .withTag("service.name", "my-service") + .withTag("elasticsearch.version", "7.0") + .start(); + span.setBaggageItem("baggage", "item"); + span.context().setDataTop("mydata", "[1,2,3]"); + List trace = Collections.singletonList(span); + + TraceMapperV0_5 traceMapper = new TraceMapperV0_5(); + CapturingByteBufferConsumer sink = new CapturingByteBufferConsumer(); + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1024, sink)); + packer.format(trace, traceMapper); + packer.flush(); + + // only top-level statements in Spock then-blocks are power assertions; + // expressions inside for-loops are not, so we only assert the critical outcomes + assertNotNull(sink.captured); + GrowableBuffer dictionaryBuffer = TraceMapperTestBridge.getDictionary(traceMapper); + ByteBuffer dictionaryBytes = dictionaryBuffer.slice(); + Map meta = new HashMap<>(); + + MessageUnpacker dictionaryUnpacker = MessagePack.newDefaultUnpacker(dictionaryBytes); + int dictionaryLength = TraceMapperTestBridge.getEncoding(traceMapper).size(); + String[] dictionary = new String[dictionaryLength]; + for (int i = 0; i < dictionary.length; ++i) { + dictionary[i] = dictionaryUnpacker.unpackString(); + } + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(sink.captured); + int traceCount = unpacker.unpackArrayHeader(); + assertEquals(1, traceCount); + for (int i = 0; i < traceCount; ++i) { + int arrayLength = unpacker.unpackArrayHeader(); + assertEquals(12, arrayLength); + String serviceName = dictionary[unpacker.unpackInt()]; + assertEquals("my-service", serviceName); + String operationName = dictionary[unpacker.unpackInt()]; + assertTrue(operationName.isEmpty()); + String resourceName = dictionary[unpacker.unpackInt()]; + assertTrue(resourceName.isEmpty()); + long traceId = unpacker.unpackLong(); + assertTrue(traceId > 0); + long spanId = unpacker.unpackLong(); + assertTrue(spanId > 0); + long parentId = unpacker.unpackLong(); + assertEquals(0, parentId); + long start = unpacker.unpackLong(); + assertTrue(start > 0); + long duration = unpacker.unpackLong(); + assertEquals(-start, duration); + int error = unpacker.unpackInt(); + assertEquals(0, error); + int metaHeader = unpacker.unpackMapHeader(); + for (int j = 0; j < metaHeader; ++j) { + String key = dictionary[unpacker.unpackInt()]; + assertNotNull(key); + String value = dictionary[unpacker.unpackInt()]; + assertNotNull(value); + meta.put(key, value); + } + int metricsHeader = unpacker.unpackMapHeader(); + for (int j = 0; j < metricsHeader; ++j) { + String key = dictionary[unpacker.unpackInt()]; + assertNotNull(key); + unpacker.skipValue(); + } + String type = dictionary[unpacker.unpackInt()]; + assertNotNull(type); + + // find the meta entry whose key contains ".mydata." and verify its value + String myDataValue = null; + for (Map.Entry entry : meta.entrySet()) { + if (entry.getKey().contains(".mydata.")) { + myDataValue = entry.getValue(); + break; + } + } + assertEquals("[1,2,3]", myDataValue); + } + + tracer.close(); + } + + static class CapturingByteBufferConsumer implements ByteBufferConsumer { + + ByteBuffer captured; + + @Override + public void accept(int messageCount, ByteBuffer buffer) { + captured = buffer; + } + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java new file mode 100644 index 00000000000..6f57f5d455b --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java @@ -0,0 +1,645 @@ +package datadog.trace.common.writer; + +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; +import static datadog.trace.common.writer.ddagent.Prioritization.FAST_LANE; +import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.ENQUEUED_FOR_SERIALIZATION; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import datadog.trace.bootstrap.instrumentation.api.SpanPostProcessor; +import datadog.trace.common.sampling.SingleSpanSampler; +import datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult; +import datadog.trace.core.CoreSpan; +import datadog.trace.core.DDSpan; +import datadog.trace.core.DDSpanContext; +import datadog.trace.core.PendingTrace; +import datadog.trace.core.monitor.HealthMetrics; +import datadog.trace.junit.utils.tabletest.PrioritySamplingConverter; +import datadog.trace.test.util.DDJavaSpecification; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.converter.ConvertWith; +import org.tabletest.junit.TableTest; + +class TraceProcessingWorkerTest extends DDJavaSpecification { + + // ------------------------------------------------------------------------- + // Helper: flush-counting payload dispatcher + // ------------------------------------------------------------------------- + + private PayloadDispatcherImpl flushCountingPayloadDispatcher(AtomicInteger flushCounter) { + PayloadDispatcherImpl dispatcher = mock(PayloadDispatcherImpl.class); + doAnswer( + inv -> { + flushCounter.incrementAndGet(); + return null; + }) + .when(dispatcher) + .flush(); + return dispatcher; + } + + // ------------------------------------------------------------------------- + // Test 1: heartbeats should be triggered automatically when enabled + // ------------------------------------------------------------------------- + + @Test + void testHeartbeatsShouldBeTriggeredAutomaticallyWhenEnabled() throws Exception { + AtomicInteger flushCount = new AtomicInteger(); + TraceProcessingWorker worker = + new TraceProcessingWorker( + 10, + mock(HealthMetrics.class), + flushCountingPayloadDispatcher(flushCount), + () -> false, + FAST_LANE, + 1, + TimeUnit.NANOSECONDS, // stop heartbeats from being throttled + null); + + // processor is started + worker.start(); + + try { + // heartbeat occurs automatically + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + if (flushCount.get() > 0) break; + Thread.sleep(50); + } + assertTrue(flushCount.get() > 0); + } finally { + // cleanup + worker.close(); + } + } + + // ------------------------------------------------------------------------- + // Test 2: heartbeats should occur at least once per second when not throttled + // ------------------------------------------------------------------------- + + @Test + void testHeartbeatsShouldOccurAtLeastOncePerSecondWhenNotThrottled() throws Exception { + AtomicInteger flushCount = new AtomicInteger(); + TraceProcessingWorker worker = + new TraceProcessingWorker( + 10, + mock(HealthMetrics.class), + flushCountingPayloadDispatcher(flushCount), + () -> false, + FAST_LANE, + 1, + TimeUnit.NANOSECONDS, // stop heartbeats from being throttled + null); + + // processor is started + worker.start(); + + try { + // heartbeat occurs automatically approximately once per second + // wait 1 second initial delay, then poll up to 1 second + Thread.sleep(1000); + long deadline = System.currentTimeMillis() + 1000; + while (System.currentTimeMillis() < deadline) { + if (flushCount.get() > 1) break; + Thread.sleep(50); + } + assertTrue(flushCount.get() > 1); + } finally { + // cleanup + worker.close(); + } + } + + // ------------------------------------------------------------------------- + // Test 3: a flush should clear the primary queue + // ------------------------------------------------------------------------- + + @Test + void testAFlushShouldClearThePrimaryQueue() { + AtomicInteger flushCount = new AtomicInteger(); + TraceProcessingWorker worker = + new TraceProcessingWorker( + 10, + mock(HealthMetrics.class), + flushCountingPayloadDispatcher(flushCount), + () -> false, + FAST_LANE, + 100, + TimeUnit.SECONDS, // prevent heartbeats from helping the flush happen + null); + + try { + // there is pending work it is completed before a flush + // processing this span will throw an exception, but it should be caught + // and not disrupt the flush + List trace = Collections.singletonList(mock(DDSpan.class)); + worker.getPrimaryQueue().offer(trace); + worker.start(); + boolean flushed = worker.flush(10, TimeUnit.SECONDS); + + // the flush succeeds, triggers a dispatch, and the queue is empty + assertTrue(flushed); + assertEquals(1, flushCount.get()); + assertTrue(worker.getPrimaryQueue().isEmpty()); + } finally { + // cleanup + worker.close(); + } + } + + // ------------------------------------------------------------------------- + // Test 4: should report failure if serialization fails + // ------------------------------------------------------------------------- + + @TableTest({ + "scenario | priority ", + "sampler drop | PrioritySampling.SAMPLER_DROP", + "user drop | PrioritySampling.USER_DROP ", + "sampler keep | PrioritySampling.SAMPLER_KEEP", + "user keep | PrioritySampling.USER_KEEP ", + "unset | PrioritySampling.UNSET " + }) + void testShouldReportFailureIfSerializationFails( + @ConvertWith(PrioritySamplingConverter.class) int priority) throws Exception { + Throwable theError = new IllegalStateException("thrown by test"); + PayloadDispatcherImpl throwingDispatcher = mock(PayloadDispatcherImpl.class); + doAnswer( + inv -> { + throw theError; + }) + .when(throwingDispatcher) + .addTrace(any()); + + AtomicInteger errorReported = new AtomicInteger(); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + // do this manually with a counter, despite mockito's lovely syntactical sugar so we don't have + // a race condition induced flaky test. All we care about is that an error was reported and that + // it was the right one + doAnswer( + inv -> { + errorReported.incrementAndGet(); + return null; + }) + .when(healthMetrics) + .onFailedSerialize(any(), any()); + + TraceProcessingWorker worker = + new TraceProcessingWorker( + 10, + healthMetrics, + throwingDispatcher, + () -> false, + FAST_LANE, + 100, + TimeUnit.SECONDS, // prevent heartbeats from helping the flush happen + null); + worker.start(); + + try { + // a trace is processed but can't be passed on + worker.publish(mock(DDSpan.class), priority, Collections.singletonList(mock(DDSpan.class))); + + // the error is reported to the monitor + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + if (errorReported.get() == 1) break; + Thread.sleep(50); + } + assertEquals(1, errorReported.get()); + } finally { + // cleanup + worker.close(); + } + } + + // ------------------------------------------------------------------------- + // Test 5: trace should be post-processed + // ------------------------------------------------------------------------- + + @Test + void testTraceShouldBePostProcessed() throws Exception { + AtomicInteger acceptedCount = new AtomicInteger(); + PayloadDispatcherImpl countingDispatcher = mock(PayloadDispatcherImpl.class); + doAnswer( + inv -> { + acceptedCount.getAndIncrement(); + return null; + }) + .when(countingDispatcher) + .addTrace(any()); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + + // Create real DDSpan instances via reflection (DDSpan.create is package-private) + Method createMethod = + DDSpan.class.getDeclaredMethod( + "create", String.class, long.class, DDSpanContext.class, List.class); + createMethod.setAccessible(true); + + DDSpanContext ctx1 = mock(DDSpanContext.class); + PendingTrace pendingTrace1 = mock(PendingTrace.class); + when(ctx1.getTraceCollector()).thenReturn(pendingTrace1); + when(pendingTrace1.getCurrentTimeNano()).thenReturn(0L); + + DDSpanContext ctx2 = mock(DDSpanContext.class); + PendingTrace pendingTrace2 = mock(PendingTrace.class); + when(ctx2.getTraceCollector()).thenReturn(pendingTrace2); + when(pendingTrace2.getCurrentTimeNano()).thenReturn(0L); + + DDSpan span1 = (DDSpan) createMethod.invoke(null, "test", 0L, ctx1, Collections.emptyList()); + DDSpan span2 = (DDSpan) createMethod.invoke(null, "test", 0L, ctx2, Collections.emptyList()); + + AtomicBoolean processedSpan1 = new AtomicBoolean(false); + AtomicBoolean processedSpan2 = new AtomicBoolean(false); + + SpanPostProcessor mockProcessor = mock(SpanPostProcessor.class); + doAnswer( + inv -> { + Object spanArg = inv.getArgument(0); + if (spanArg == span1) processedSpan1.set(true); + if (spanArg == span2) processedSpan2.set(true); + return null; + }) + .when(mockProcessor) + .process(any(), any()); + + SpanPostProcessor.Holder.INSTANCE = mockProcessor; + + TraceProcessingWorker worker = + new TraceProcessingWorker( + 10, + healthMetrics, + countingDispatcher, + () -> false, + FAST_LANE, + 100, + TimeUnit.SECONDS, + null); + worker.start(); + + try { + // traces are submitted + List trace = new ArrayList<>(); + trace.add(span1); + trace.add(span2); + worker.publish(span1, SAMPLER_KEEP, trace); + worker.publish(span2, SAMPLER_KEEP, trace); + + // traces are passed through unless rejected on submission + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + if (processedSpan1.get() && processedSpan2.get()) break; + Thread.sleep(50); + } + assertTrue(processedSpan1.get()); + assertTrue(processedSpan2.get()); + } finally { + // cleanup + SpanPostProcessor.Holder.INSTANCE = SpanPostProcessor.Holder.NOOP; + worker.close(); + } + } + + // ------------------------------------------------------------------------- + // Test 6: traces should be processed + // ------------------------------------------------------------------------- + + @TableTest({ + "scenario | priority | traceCount", + "sampler drop x1 | PrioritySampling.SAMPLER_DROP | 1 ", + "user drop x1 | PrioritySampling.USER_DROP | 1 ", + "sampler keep x1 | PrioritySampling.SAMPLER_KEEP | 1 ", + "user keep x1 | PrioritySampling.USER_KEEP | 1 ", + "unset x1 | PrioritySampling.UNSET | 1 ", + "sampler drop x10 | PrioritySampling.SAMPLER_DROP | 10 ", + "user drop x10 | PrioritySampling.USER_DROP | 10 ", + "sampler keep x10 | PrioritySampling.SAMPLER_KEEP | 10 ", + "user keep x10 | PrioritySampling.USER_KEEP | 10 ", + "unset x10 | PrioritySampling.UNSET | 10 ", + "sampler drop x20 | PrioritySampling.SAMPLER_DROP | 20 ", + "user drop x20 | PrioritySampling.USER_DROP | 20 ", + "sampler keep x20 | PrioritySampling.SAMPLER_KEEP | 20 ", + "user keep x20 | PrioritySampling.USER_KEEP | 20 ", + "unset x20 | PrioritySampling.UNSET | 20 ", + "sampler drop x100 | PrioritySampling.SAMPLER_DROP | 100 ", + "user drop x100 | PrioritySampling.USER_DROP | 100 ", + "sampler keep x100 | PrioritySampling.SAMPLER_KEEP | 100 ", + "user keep x100 | PrioritySampling.USER_KEEP | 100 ", + "unset x100 | PrioritySampling.UNSET | 100 " + }) + void testTracesShouldBeProcessed( + @ConvertWith(PrioritySamplingConverter.class) int priority, int traceCount) throws Exception { + AtomicInteger acceptedCount = new AtomicInteger(); + PayloadDispatcherImpl countingDispatcher = mock(PayloadDispatcherImpl.class); + doAnswer( + inv -> { + acceptedCount.getAndIncrement(); + return null; + }) + .when(countingDispatcher) + .addTrace(any()); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + TraceProcessingWorker worker = + new TraceProcessingWorker( + 10, + healthMetrics, + countingDispatcher, + () -> false, + FAST_LANE, + 100, + TimeUnit.SECONDS, // prevent heartbeats from helping the flush happen + null); + worker.start(); + + try { + // traces are submitted + int submitted = 0; + for (int i = 0; i < traceCount; ++i) { + PublishResult publishResult = + worker.publish( + mock(DDSpan.class), priority, Collections.singletonList(mock(DDSpan.class))); + submitted += publishResult == ENQUEUED_FOR_SERIALIZATION ? 1 : 0; + } + + // traces are passed through unless rejected on submission + int expectedSubmitted = submitted; + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + if (expectedSubmitted == acceptedCount.get()) break; + Thread.sleep(50); + } + assertEquals(expectedSubmitted, acceptedCount.get()); + } finally { + // cleanup + worker.close(); + } + } + + // ------------------------------------------------------------------------- + // Test 7: flush of full queue after worker thread stopped will not flush but will return + // ------------------------------------------------------------------------- + + @Test + void testFlushOfFullQueueAfterWorkerThreadStoppedWillNotFlushButWillReturn() { + PayloadDispatcherImpl countingDispatcher = mock(PayloadDispatcherImpl.class); + HealthMetrics healthMetrics = mock(HealthMetrics.class); + TraceProcessingWorker worker = + new TraceProcessingWorker( + 10, + healthMetrics, + countingDispatcher, + () -> false, + FAST_LANE, + 100, + TimeUnit.SECONDS, + null); + worker.start(); + worker.close(); + + while (worker.getPrimaryQueue().offer(Collections.singletonList(mock(DDSpan.class)))) { + // fill the queue + } + + boolean flushed = worker.flush(1, TimeUnit.SECONDS); + assertFalse(flushed); + } + + // ------------------------------------------------------------------------- + // Test 8: send unsampled traces - dropping policy is active + // ------------------------------------------------------------------------- + + @TableTest({ + "scenario | priority | traceCount | acceptedTraces | acceptedSpans | sampledSingleSpans | spanCount", + "sampler drop 1 trace 1 span | PrioritySampling.SAMPLER_DROP | 1 | 1 | 1 | 1 | 1 ", + "user drop 1 trace 2 spans | PrioritySampling.USER_DROP | 1 | 1 | 1 | 1 | 2 ", + "sampler drop 1 trace 3 spans | PrioritySampling.SAMPLER_DROP | 1 | 1 | 2 | 2 | 3 ", + "user drop 1 trace 4 spans | PrioritySampling.USER_DROP | 1 | 1 | 2 | 2 | 4 ", + "sampler drop 1 trace 5 spans | PrioritySampling.SAMPLER_DROP | 1 | 1 | 3 | 3 | 5 ", + "user drop 2 traces 1 span | PrioritySampling.USER_DROP | 2 | 1 | 1 | 1 | 1 ", + "sampler drop 2 traces 2 spans | PrioritySampling.SAMPLER_DROP | 2 | 2 | 2 | 2 | 2 ", + "user drop 2 traces 3 spans | PrioritySampling.USER_DROP | 2 | 2 | 3 | 3 | 3 ", + "sampler drop 2 traces 4 spans | PrioritySampling.SAMPLER_DROP | 2 | 2 | 4 | 4 | 4 ", + "user drop 2 traces 5 spans | PrioritySampling.USER_DROP | 2 | 2 | 5 | 5 | 5 ", + "sampler drop 10 traces 1 span | PrioritySampling.SAMPLER_DROP | 10 | 5 | 5 | 5 | 1 ", + "user drop 10 traces 2 spans | PrioritySampling.USER_DROP | 10 | 10 | 10 | 10 | 2 ", + "sampler drop 10 traces 3 spans | PrioritySampling.SAMPLER_DROP | 10 | 10 | 15 | 15 | 3 ", + "user drop 10 traces 4 spans | PrioritySampling.USER_DROP | 10 | 10 | 20 | 20 | 4 ", + "sampler drop 10 traces 5 spans | PrioritySampling.SAMPLER_DROP | 10 | 10 | 25 | 25 | 5 ", + "sampler keep 1 trace 1 span | PrioritySampling.SAMPLER_KEEP | 1 | 1 | 1 | 0 | 1 ", + "user keep 1 trace 2 spans | PrioritySampling.USER_KEEP | 1 | 1 | 2 | 0 | 2 ", + "sampler keep 1 trace 3 spans | PrioritySampling.SAMPLER_KEEP | 1 | 1 | 3 | 0 | 3 ", + "user keep 1 trace 4 spans | PrioritySampling.USER_KEEP | 1 | 1 | 4 | 0 | 4 ", + "sampler keep 1 trace 5 spans | PrioritySampling.SAMPLER_KEEP | 1 | 1 | 5 | 0 | 5 ", + "user keep 2 traces 1 span | PrioritySampling.USER_KEEP | 2 | 2 | 2 | 0 | 1 ", + "sampler keep 2 traces 2 spans | PrioritySampling.SAMPLER_KEEP | 2 | 2 | 4 | 0 | 2 ", + "user keep 2 traces 3 spans | PrioritySampling.USER_KEEP | 2 | 2 | 6 | 0 | 3 ", + "sampler keep 2 traces 4 spans | PrioritySampling.SAMPLER_KEEP | 2 | 2 | 8 | 0 | 4 ", + "user keep 2 traces 5 spans | PrioritySampling.USER_KEEP | 2 | 2 | 10 | 0 | 5 ", + "sampler keep 10 traces 1 span | PrioritySampling.SAMPLER_KEEP | 10 | 10 | 10 | 0 | 1 ", + "user keep 10 traces 2 spans | PrioritySampling.USER_KEEP | 10 | 10 | 20 | 0 | 2 ", + "sampler keep 10 traces 3 spans | PrioritySampling.SAMPLER_KEEP | 10 | 10 | 30 | 0 | 3 ", + "user keep 10 traces 4 spans | PrioritySampling.USER_KEEP | 10 | 10 | 40 | 0 | 4 ", + "sampler keep 10 traces 5 spans | PrioritySampling.SAMPLER_KEEP | 10 | 10 | 50 | 0 | 5 " + }) + void testSendUnsampledTracesDroppingPolicyIsActive( + @ConvertWith(PrioritySamplingConverter.class) int priority, + int traceCount, + int acceptedTraces, + int acceptedSpans, + int sampledSingleSpans, + int spanCount) + throws Exception { + HealthMetrics healthMetrics = mock(HealthMetrics.class); + AtomicInteger acceptedCount = new AtomicInteger(); + AtomicInteger acceptedSpanCount = new AtomicInteger(); + PayloadDispatcherImpl countingDispatcher = mock(PayloadDispatcherImpl.class); + doAnswer( + inv -> { + List traceList = inv.getArgument(0); + acceptedSpanCount.getAndAdd(traceList.size()); + acceptedCount.getAndIncrement(); + return null; + }) + .when(countingDispatcher) + .addTrace(any()); + + AtomicInteger sampledSpansCount = new AtomicInteger(); + // drop every other span + SingleSpanSampler singleSpanSampler = + new SingleSpanSampler() { + int counter = 0; + + @Override + public > boolean setSamplingPriority(T span) { + if (counter++ % 2 == 0) { + sampledSpansCount.incrementAndGet(); + return true; + } + return false; + } + }; + + TraceProcessingWorker worker = + new TraceProcessingWorker( + 10, + healthMetrics, + countingDispatcher, + () -> true, + FAST_LANE, + 100, + TimeUnit.SECONDS, + singleSpanSampler); + worker.start(); + + try { + // traces are submitted + for (int i = 0; i < traceCount; ++i) { + List trace = new ArrayList<>(); + for (int j = 0; j < spanCount; j++) { + trace.add(mock(DDSpan.class)); + } + worker.publish(trace.get(0), priority, trace); + } + + // traces are passed through unless rejected on submission + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + if (acceptedTraces == acceptedCount.get() + && acceptedSpans == acceptedSpanCount.get() + && sampledSingleSpans == sampledSpansCount.get()) break; + Thread.sleep(50); + } + assertEquals(acceptedTraces, acceptedCount.get()); + assertEquals(acceptedSpans, acceptedSpanCount.get()); + assertEquals(sampledSingleSpans, sampledSpansCount.get()); + } finally { + // cleanup + worker.close(); + } + } + + // ------------------------------------------------------------------------- + // Test 9: send unsampled traces - dropping policy is inactive + // ------------------------------------------------------------------------- + + @TableTest({ + "scenario | priority | traceCount | expectedChunks | expectedSpans | sampledSingleSpans | spanCount", + "sampler drop 1 trace 1 span | PrioritySampling.SAMPLER_DROP | 1 | 1 | 1 | 1 | 1 ", + "user drop 1 trace 2 spans | PrioritySampling.USER_DROP | 1 | 2 | 2 | 1 | 2 ", + "sampler drop 1 trace 3 spans | PrioritySampling.SAMPLER_DROP | 1 | 2 | 3 | 2 | 3 ", + "user drop 1 trace 4 spans | PrioritySampling.USER_DROP | 1 | 2 | 4 | 2 | 4 ", + "sampler drop 1 trace 5 spans | PrioritySampling.SAMPLER_DROP | 1 | 2 | 5 | 3 | 5 ", + "user drop 2 traces 1 span | PrioritySampling.USER_DROP | 2 | 2 | 2 | 1 | 1 ", + "sampler drop 2 traces 2 spans | PrioritySampling.SAMPLER_DROP | 2 | 4 | 4 | 2 | 2 ", + "user drop 2 traces 3 spans | PrioritySampling.USER_DROP | 2 | 4 | 6 | 3 | 3 ", + "sampler drop 2 traces 4 spans | PrioritySampling.SAMPLER_DROP | 2 | 4 | 8 | 4 | 4 ", + "user drop 2 traces 5 spans | PrioritySampling.USER_DROP | 2 | 4 | 10 | 5 | 5 ", + "user drop 10 traces 1 span | PrioritySampling.USER_DROP | 10 | 10 | 10 | 5 | 1 ", + "sampler drop 10 traces 2 spans | PrioritySampling.SAMPLER_DROP | 10 | 20 | 20 | 10 | 2 ", + "user drop 10 traces 3 spans | PrioritySampling.USER_DROP | 10 | 20 | 30 | 15 | 3 ", + "sampler drop 10 traces 4 spans | PrioritySampling.SAMPLER_DROP | 10 | 20 | 40 | 20 | 4 ", + "user drop 10 traces 5 spans | PrioritySampling.USER_DROP | 10 | 20 | 50 | 25 | 5 ", + "sampler keep 1 trace 1 span | PrioritySampling.SAMPLER_KEEP | 1 | 1 | 1 | 0 | 1 ", + "user keep 1 trace 2 spans | PrioritySampling.USER_KEEP | 1 | 1 | 2 | 0 | 2 ", + "sampler keep 1 trace 3 spans | PrioritySampling.SAMPLER_KEEP | 1 | 1 | 3 | 0 | 3 ", + "user keep 1 trace 4 spans | PrioritySampling.USER_KEEP | 1 | 1 | 4 | 0 | 4 ", + "sampler keep 1 trace 5 spans | PrioritySampling.SAMPLER_KEEP | 1 | 1 | 5 | 0 | 5 ", + "user keep 2 traces 1 span | PrioritySampling.USER_KEEP | 2 | 2 | 2 | 0 | 1 ", + "sampler keep 2 traces 2 spans | PrioritySampling.SAMPLER_KEEP | 2 | 2 | 4 | 0 | 2 ", + "user keep 2 traces 3 spans | PrioritySampling.USER_KEEP | 2 | 2 | 6 | 0 | 3 ", + "sampler keep 2 traces 4 spans | PrioritySampling.SAMPLER_KEEP | 2 | 2 | 8 | 0 | 4 ", + "user keep 2 traces 5 spans | PrioritySampling.USER_KEEP | 2 | 2 | 10 | 0 | 5 ", + "sampler keep 10 traces 1 span | PrioritySampling.SAMPLER_KEEP | 10 | 10 | 10 | 0 | 1 ", + "user keep 10 traces 2 spans | PrioritySampling.USER_KEEP | 10 | 10 | 20 | 0 | 2 ", + "sampler keep 10 traces 3 spans | PrioritySampling.SAMPLER_KEEP | 10 | 10 | 30 | 0 | 3 ", + "user keep 10 traces 4 spans | PrioritySampling.USER_KEEP | 10 | 10 | 40 | 0 | 4 ", + "sampler keep 10 traces 5 spans | PrioritySampling.SAMPLER_KEEP | 10 | 10 | 50 | 0 | 5 " + }) + void testSendUnsampledTracesDroppingPolicyIsInactive( + @ConvertWith(PrioritySamplingConverter.class) int priority, + int traceCount, + int expectedChunks, + int expectedSpans, + int sampledSingleSpans, + int spanCount) + throws Exception { + HealthMetrics healthMetrics = mock(HealthMetrics.class); + AtomicInteger chunksCount = new AtomicInteger(); + AtomicInteger spansCount = new AtomicInteger(); + PayloadDispatcherImpl countingDispatcher = mock(PayloadDispatcherImpl.class); + doAnswer( + inv -> { + List traceList = inv.getArgument(0); + spansCount.getAndAdd(traceList.size()); + chunksCount.getAndIncrement(); + return null; + }) + .when(countingDispatcher) + .addTrace(any()); + + AtomicInteger sampledSpansCount = new AtomicInteger(); + // drop every other span + SingleSpanSampler singleSpanSampler = + new SingleSpanSampler() { + int counter = 0; + + @Override + public > boolean setSamplingPriority(T span) { + if (counter++ % 2 == 0) { + sampledSpansCount.incrementAndGet(); + return true; + } + return false; + } + }; + + TraceProcessingWorker worker = + new TraceProcessingWorker( + 10, + healthMetrics, + countingDispatcher, + () -> false, + FAST_LANE, + 100, + TimeUnit.SECONDS, + singleSpanSampler); + worker.start(); + + try { + // traces are submitted + for (int i = 0; i < traceCount; ++i) { + List trace = new ArrayList<>(); + for (int j = 0; j < spanCount; j++) { + trace.add(mock(DDSpan.class)); + } + worker.publish(trace.get(0), priority, trace); + } + + // traces are passed through unless rejected on submission + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + if (expectedChunks == chunksCount.get() + && expectedSpans == spansCount.get() + && sampledSingleSpans == sampledSpansCount.get()) break; + Thread.sleep(50); + } + assertEquals(expectedChunks, chunksCount.get()); + assertEquals(expectedSpans, spansCount.get()); + assertEquals(sampledSingleSpans, sampledSpansCount.get()); + } finally { + // cleanup + worker.close(); + } + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/WriterFactoryTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/WriterFactoryTest.java new file mode 100644 index 00000000000..bb52f69806a --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/WriterFactoryTest.java @@ -0,0 +1,295 @@ +package datadog.trace.common.writer; + +import static datadog.trace.api.config.TracerConfig.PRIORITIZATION_TYPE; +import static java.util.Collections.singletonList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.Config; +import datadog.trace.api.config.OtlpConfig; +import datadog.trace.api.intake.TrackType; +import datadog.trace.common.sampling.Sampler; +import datadog.trace.common.writer.ddagent.Prioritization; +import datadog.trace.core.monitor.HealthMetrics; +import datadog.trace.test.util.DDJavaSpecification; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import okhttp3.Call; +import okhttp3.HttpUrl; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.tabletest.junit.TableTest; + +class WriterFactoryTest extends DDJavaSpecification { + + @TableTest({ + "scenario | configuredType | hasEvpProxy | evpProxySupportsCompression | isCiVisibilityAgentlessEnabled | expectedWriterClass | expectedApiClass | isCompressionEnabled", + "LoggingWriter agentless | LoggingWriter | true | false | true | datadog.trace.common.writer.LoggingWriter | | false ", + "PrintingWriter agentless | PrintingWriter | true | false | true | datadog.trace.common.writer.PrintingWriter | | false ", + "TraceStructureWriter agentless | TraceStructureWriter | true | false | true | datadog.trace.common.writer.TraceStructureWriter | | false ", + "MultiWriter agentless | 'MultiWriter:LoggingWriter,PrintingWriter' | true | false | true | datadog.trace.common.writer.MultiWriter | | false ", + "DDIntakeWriter evp agentless | DDIntakeWriter | true | false | true | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDIntakeApi | true ", + "DDIntakeWriter evp not agentless | DDIntakeWriter | true | false | false | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDEvpProxyApi | false ", + "DDIntakeWriter no evp agentless | DDIntakeWriter | false | false | true | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDIntakeApi | true ", + "DDIntakeWriter no evp not agentless | DDIntakeWriter | false | false | false | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDIntakeApi | true ", + "DDAgentWriter evp agentless | DDAgentWriter | true | false | true | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDIntakeApi | true ", + "DDAgentWriter evp not agentless | DDAgentWriter | true | false | false | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDEvpProxyApi | false ", + "DDAgentWriter evp compression not agentless | DDAgentWriter | true | true | false | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDEvpProxyApi | true ", + "DDAgentWriter no evp agentless | DDAgentWriter | false | false | true | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDIntakeApi | true ", + "DDAgentWriter no evp not agentless | DDAgentWriter | false | false | false | datadog.trace.common.writer.DDAgentWriter | datadog.trace.common.writer.ddagent.DDAgentApi | false ", + "not-found evp agentless | 'not-found' | true | false | true | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDIntakeApi | true ", + "not-found evp not agentless | 'not-found' | true | false | false | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDEvpProxyApi | false ", + "not-found no evp agentless | 'not-found' | false | false | true | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDIntakeApi | true ", + "not-found no evp not agentless | 'not-found' | false | false | false | datadog.trace.common.writer.DDAgentWriter | datadog.trace.common.writer.ddagent.DDAgentApi | false " + }) + void testWriterCreationForCiVisibility( + String configuredType, + boolean hasEvpProxy, + boolean evpProxySupportsCompression, + boolean isCiVisibilityAgentlessEnabled, + Class expectedWriterClass, + Class expectedApiClass, + boolean isCompressionEnabled) + throws Exception { + Config config = mock(Config.class); + when(config.getApiKey()).thenReturn("my-api-key"); + when(config.getAgentUrl()).thenReturn("http://my-agent.url"); + //noinspection unchecked + doReturn(Prioritization.FAST_LANE) + .when(config) + .getEnumValue( + eq(PRIORITIZATION_TYPE), + (Class) any(Class.class), + any(Prioritization.class)); + when(config.isTracerMetricsEnabled()).thenReturn(true); + when(config.isCiVisibilityEnabled()).thenReturn(true); + when(config.isCiVisibilityCodeCoverageEnabled()).thenReturn(false); + when(config.isCiVisibilityAgentlessEnabled()).thenReturn(isCiVisibilityAgentlessEnabled); + + // Mock agent info response + Response response = + buildHttpResponse( + hasEvpProxy, evpProxySupportsCompression, HttpUrl.parse("http://my-agent.url/info")); + + // Mock HTTP client that simulates delayed response for async feature discovery + Call mockCall = mock(Call.class); + OkHttpClient mockHttpClient = mock(OkHttpClient.class); + when(mockCall.execute()) + .thenAnswer( + inv -> { + // Add a delay + Thread.sleep(400); + return response; + }); + when(mockHttpClient.newCall(any(Request.class))).thenReturn(mockCall); + + // Create SharedCommunicationObjects with mocked HTTP client + SharedCommunicationObjects sharedComm = new SharedCommunicationObjects(); + sharedComm.agentHttpClient = mockHttpClient; + sharedComm.agentUrl = HttpUrl.parse("http://my-agent.url"); + sharedComm.createRemaining(config); + Sampler sampler = mock(Sampler.class); + + Writer writer = + WriterFactory.createWriter( + config, sharedComm, sampler, null, HealthMetrics.NO_OP, configuredType); + + List> expectedApiClasses = + expectedApiClass != null ? singletonList(expectedApiClass) : null; + Collection apis; + List> apiClasses; + if (expectedApiClasses != null) { + apis = ((RemoteWriter) writer).getApis(); + apiClasses = apis.stream().map(Object::getClass).collect(Collectors.toList()); + } else { + apis = java.util.Collections.emptyList(); + apiClasses = java.util.Collections.emptyList(); + } + + assertEquals(expectedWriterClass, writer.getClass()); + assertTrue(expectedApiClasses == null || apiClasses.equals(expectedApiClasses)); + assertTrue( + expectedApiClasses == null + || apis.stream().allMatch(api -> api.isCompressionEnabled() == isCompressionEnabled)); + } + + @TableTest({ + "scenario | configuredType | agentRunning | hasEvpProxy | isLlmObsAgentlessEnabled | expectedWriterClass | expectedLlmObsApiClass ", + "evp proxy not agentless | DDIntakeWriter | true | true | false | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDEvpProxyApi", + "no evp not agentless | DDIntakeWriter | true | false | false | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDIntakeApi ", + "agent not running not agentless | DDIntakeWriter | false | false | false | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDIntakeApi ", + "evp proxy agentless | DDIntakeWriter | true | true | true | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDIntakeApi ", + "no evp agentless | DDIntakeWriter | true | false | true | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDIntakeApi ", + "agent not running agentless | DDIntakeWriter | false | false | true | datadog.trace.common.writer.DDIntakeWriter | datadog.trace.common.writer.ddintake.DDIntakeApi " + }) + void testWriterCreationForLlmObservability( + String configuredType, + boolean agentRunning, + boolean hasEvpProxy, + boolean isLlmObsAgentlessEnabled, + Class expectedWriterClass, + Class expectedLlmObsApiClass) + throws Exception { + Config config = mock(Config.class); + when(config.getApiKey()).thenReturn("my-api-key"); + when(config.getAgentUrl()).thenReturn("http://my-agent.url"); + //noinspection unchecked + doReturn(Prioritization.FAST_LANE) + .when(config) + .getEnumValue( + eq(PRIORITIZATION_TYPE), + (Class) any(Class.class), + any(Prioritization.class)); + when(config.isTracerMetricsEnabled()).thenReturn(true); + when(config.isLlmObsEnabled()).thenReturn(true); + when(config.isLlmObsAgentlessEnabled()).thenReturn(isLlmObsAgentlessEnabled); + + // Mock agent info response + Response response; + if (agentRunning) { + response = buildHttpResponse(hasEvpProxy, true, HttpUrl.parse("http://my-agent.url/info")); + } else { + response = buildHttpResponseNotOk(HttpUrl.parse("http://my-agent.url/info")); + } + + // Mock HTTP client that simulates delayed response for async feature discovery + Call mockCall = mock(Call.class); + OkHttpClient mockHttpClient = mock(OkHttpClient.class); + when(mockCall.execute()) + .thenAnswer( + inv -> { + // Add a delay + Thread.sleep(400); + return response; + }); + when(mockHttpClient.newCall(any(Request.class))).thenReturn(mockCall); + + // Create SharedCommunicationObjects with mocked HTTP client + SharedCommunicationObjects sharedComm = new SharedCommunicationObjects(); + sharedComm.agentHttpClient = mockHttpClient; + sharedComm.agentUrl = HttpUrl.parse("http://my-agent.url"); + sharedComm.createRemaining(config); + Sampler sampler = mock(Sampler.class); + + Writer writer = + WriterFactory.createWriter( + config, sharedComm, sampler, null, HealthMetrics.NO_OP, configuredType); + List> llmObsApiClasses = + ((RemoteWriter) writer) + .getApis().stream() + .filter( + api -> { + try { + Field trackTypeField = api.getClass().getDeclaredField("trackType"); + trackTypeField.setAccessible(true); + return trackTypeField.get(api) == TrackType.LLMOBS; + } catch (Exception e) { + return false; + } + }) + .map(Object::getClass) + .collect(Collectors.toList()); + + assertEquals(expectedWriterClass, writer.getClass()); + assertEquals(singletonList(expectedLlmObsApiClass), llmObsApiClasses); + } + + @TableTest({ + "scenario | protocol | compression | endpoint | expectedSenderClass | expectedUrl | expectedGzip", + "http no gzip | HTTP_PROTOBUF | NONE | 'http://otel-collector:4318/v1/traces' | datadog.trace.core.otlp.common.OtlpHttpSender | 'http://otel-collector:4318/v1/traces' | false ", + "http gzip | HTTP_PROTOBUF | GZIP | 'http://otel-collector:4318/v1/traces' | datadog.trace.core.otlp.common.OtlpHttpSender | 'http://otel-collector:4318/v1/traces' | true ", + "grpc no gzip | GRPC | NONE | 'http://otel-collector:4317' | datadog.trace.core.otlp.common.OtlpGrpcSender | 'http://otel-collector:4317/opentelemetry.proto.collector.trace.v1.TraceService/Export' | false " + }) + void testWriterCreationForOtlpWriter( + OtlpConfig.Protocol protocol, + OtlpConfig.Compression compression, + String endpoint, + Class expectedSenderClass, + String expectedUrl, + boolean expectedGzip) + throws Exception { + Config config = mock(Config.class); + Map headers = new HashMap<>(); + headers.put("api-key", "secret"); + + when(config.getTraceFlushIntervalSeconds()).thenReturn(1.0f); + when(config.getOtlpTracesEndpoint()).thenReturn(endpoint); + when(config.getOtlpTracesHeaders()).thenReturn(headers); + when(config.getOtlpTracesProtocol()).thenReturn(protocol); + when(config.getOtlpTracesCompression()).thenReturn(compression); + when(config.getOtlpTracesTimeout()).thenReturn(5000); + + // OTLP branch in WriterFactory does not consult sharedComm or sampler, so nulls are safe here. + Writer writer = + WriterFactory.createWriter(config, null, null, null, HealthMetrics.NO_OP, "OtlpWriter"); + Object sender = readField(writer, "sender"); + + assertEquals(OtlpWriter.class, writer.getClass()); + assertEquals(expectedSenderClass, sender.getClass()); + assertEquals(expectedUrl, readField(sender, "url").toString()); + assertEquals(headers, readField(sender, "headers")); + assertEquals(expectedGzip, readField(sender, "gzip")); + + writer.close(); + } + + private static Response buildHttpResponse( + boolean hasEvpProxy, boolean evpProxySupportsCompression, HttpUrl agentUrl) { + List endpoints = new ArrayList<>(); + if (hasEvpProxy && evpProxySupportsCompression) { + endpoints.add(DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT); + } else if (hasEvpProxy) { + endpoints.add(DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT); + } else { + endpoints.add(DDAgentFeaturesDiscovery.V04_ENDPOINT); + } + + StringBuilder endpointsJson = new StringBuilder("["); + for (int i = 0; i < endpoints.size(); i++) { + if (i > 0) endpointsJson.append(","); + endpointsJson.append("\"").append(endpoints.get(i)).append("\""); + } + endpointsJson.append("]"); + String json = "{\"version\":\"7.40.0\",\"endpoints\":" + endpointsJson + "}"; + + return new Response.Builder() + .code(200) + .message("OK") + .protocol(Protocol.HTTP_1_1) + .request(new Request.Builder().url(agentUrl.resolve("/info")).build()) + .body(ResponseBody.create(MediaType.parse("application/json"), json)) + .build(); + } + + private static Response buildHttpResponseNotOk(HttpUrl agentUrl) { + return new Response.Builder() + .code(500) + .message("ERROR") + .protocol(Protocol.HTTP_1_1) + .request(new Request.Builder().url(agentUrl.resolve("/info")).build()) + .body(ResponseBody.create(MediaType.parse("application/json"), "")) + .build(); + } + + private static Object readField(Object instance, String fieldName) throws Exception { + Field field = instance.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(instance); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java index 15483b94c95..87496e430df 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java @@ -2,13 +2,10 @@ import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.DD_MEASURED; import static datadog.trace.common.writer.TraceGenerator.generateRandomTraces; -import static datadog.trace.common.writer.ddagent.PayloadVerifiers.assertEqualsWithNullAsEmpty; -import static datadog.trace.common.writer.ddagent.PayloadVerifiers.unpackNumber; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.params.provider.Arguments.arguments; import datadog.communication.serialization.ByteBufferConsumer; @@ -21,49 +18,63 @@ import datadog.trace.api.ProcessTags; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.common.writer.Payload; -import datadog.trace.common.writer.TraceGenerator.PojoSpan; +import datadog.trace.common.writer.TraceGenerator; import datadog.trace.core.DDSpanContext; -import datadog.trace.test.junit.utils.config.WithConfigExtension; +import datadog.trace.test.util.DDJavaSpecification; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Stream; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.msgpack.core.MessageFormat; import org.msgpack.core.MessagePack; import org.msgpack.core.MessageUnpacker; +import org.tabletest.junit.TableTest; -@ExtendWith(WithConfigExtension.class) -class TraceMapperV04PayloadTest { +class TraceMapperV04PayloadTest extends DDJavaSpecification { - // Keep the ProcessTags static in sync with the (per-test rebuilt) Config, the way DDSpecification - // did for the original Spock tests. Runs after WithConfigExtension has rebuilt Config. - @BeforeEach - void syncProcessTags() { - ProcessTags.reset(Config.get()); - } - - @ParameterizedTest(name = "buffer={0} traces={1} lowCardinality={2}") - @MethodSource("tracesWrittenCorrectlyArguments") - void tracesWrittenCorrectly(int bufferSize, int traceCount, boolean lowCardinality) { - List> traces = generateRandomTraces(traceCount, lowCardinality); + @TableTest({ + "scenario | bufferSize | traceCount | lowCardinality", + "0 traces low card | 20480 | 0 | true ", + "1 trace low card 20k | 20480 | 1 | true ", + "1 trace low card 30k | 30720 | 1 | true ", + "2 traces low card | 30720 | 2 | true ", + "0 traces high card | 20480 | 0 | false ", + "1 trace high card 20k | 20480 | 1 | false ", + "1 trace high card 30k | 30720 | 1 | false ", + "2 traces high card | 30720 | 2 | false ", + "0 traces 100k low | 102400 | 0 | true ", + "1 trace 100k low | 102400 | 1 | true ", + "10 traces 100k low | 102400 | 10 | true ", + "100 traces 100k low | 102400 | 100 | true ", + "1000 traces 100k low | 102400 | 1000 | true ", + "0 traces 100k high | 102400 | 0 | false ", + "1 trace 100k high | 102400 | 1 | false ", + "10 traces 100k high | 102400 | 10 | false ", + "100 traces 100k high | 102400 | 100 | false ", + "1000 traces 100k high | 102400 | 1000 | false " + }) + void testTracesWrittenCorrectly(int bufferSize, int traceCount, boolean lowCardinality) { + List> traces = generateRandomTraces(traceCount, lowCardinality); TraceMapperV0_4 traceMapper = new TraceMapperV0_4(); PayloadVerifier verifier = new PayloadVerifier(traces, traceMapper); MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(bufferSize, verifier)); + boolean tracesFitInBuffer = true; - for (List trace : traces) { + for (List trace : traces) { if (!packer.format(trace, traceMapper)) { verifier.skipLargeTrace(); tracesFitInBuffer = false; - // in the real like the mapper is always reset each trace. + // in the real life the mapper is always reset each trace. // here we need to force it when we fail since the buffer will be reset as well traceMapper.reset(); } @@ -75,34 +86,18 @@ void tracesWrittenCorrectly(int bufferSize, int traceCount, boolean lowCardinali } } - private static Stream tracesWrittenCorrectlyArguments() { + static Stream testFull64BitTraceAndSpanIdentifiersArguments() { return Stream.of( - arguments(20 << 10, 0, true), - arguments(20 << 10, 1, true), - arguments(30 << 10, 1, true), - arguments(30 << 10, 2, true), - arguments(20 << 10, 0, false), - arguments(20 << 10, 1, false), - arguments(30 << 10, 1, false), - arguments(30 << 10, 2, false), - arguments(100 << 10, 0, true), - arguments(100 << 10, 1, true), - arguments(100 << 10, 10, true), - arguments(100 << 10, 100, true), - arguments(100 << 10, 1000, true), - arguments(100 << 10, 0, false), - arguments(100 << 10, 1, false), - arguments(100 << 10, 10, false), - arguments(100 << 10, 100, false), - arguments(100 << 10, 1000, false)); + arguments(DD64bTraceId.ONE, 2L, 3L), + arguments(DD64bTraceId.MAX, 2L, 3L), + arguments(DD64bTraceId.from(-10), -11L, -12L)); } - @ParameterizedTest(name = "{0}") - @MethodSource("fullSixtyFourBitTraceAndSpanIdentifiersArguments") - void fullSixtyFourBitTraceAndSpanIdentifiers( - String scenario, DDTraceId traceId, long spanId, long parentId) { - PojoSpan span = - new PojoSpan( + @ParameterizedTest + @MethodSource("testFull64BitTraceAndSpanIdentifiersArguments") + void testFull64BitTraceAndSpanIdentifiers(DDTraceId traceId, long spanId, long parentId) { + TraceGenerator.PojoSpan span = + new TraceGenerator.PojoSpan( "service", "operation", "resource", @@ -119,7 +114,8 @@ void fullSixtyFourBitTraceAndSpanIdentifiers( 0, 0, "origin"); - List> traces = Collections.singletonList(Collections.singletonList(span)); + List> traces = + Collections.singletonList(Collections.singletonList(span)); TraceMapperV0_4 traceMapper = new TraceMapperV0_4(); PayloadVerifier verifier = new PayloadVerifier(traces, traceMapper); MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(20 << 10, verifier)); @@ -130,17 +126,10 @@ void fullSixtyFourBitTraceAndSpanIdentifiers( verifier.verifyTracesConsumed(); } - private static Stream fullSixtyFourBitTraceAndSpanIdentifiersArguments() { - return Stream.of( - arguments("ONE", DD64bTraceId.ONE, 2L, 3L), - arguments("MAX", DD64bTraceId.MAX, 2L, 3L), - arguments("negative", DD64bTraceId.from(-10), -11L, -12L)); - } - @Test - void metaStructSupport() { - PojoSpan span = - new PojoSpan( + void testMetaStructSupport() { + TraceGenerator.PojoSpan span = + new TraceGenerator.PojoSpan( "service", "operation", "resource", @@ -157,31 +146,34 @@ void metaStructSupport() { 0, 0, "origin"); - List> stack = new ArrayList<>(); - for (StackTraceElement element : Thread.currentThread().getStackTrace()) { - Map frame = new HashMap<>(); - frame.put("file", element.getFileName() != null ? element.getFileName() : ""); - frame.put("class_name", element.getClassName() != null ? element.getClassName() : ""); - frame.put("function", element.getMethodName() != null ? element.getMethodName() : ""); - stack.add(frame); + + StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); + List> stackList = new ArrayList<>(); + for (StackTraceElement element : stackTrace) { + Map entry = new HashMap<>(); + entry.put("file", element.getFileName() != null ? element.getFileName() : ""); + entry.put("class_name", element.getClassName()); + entry.put("function", element.getMethodName()); + stackList.add(entry); } - span.setMetaStruct("stack", stack); - List> traces = Collections.singletonList(Collections.singletonList(span)); + span.setMetaStruct("stack", stackList); + + List> traces = + Collections.singletonList(Collections.singletonList(span)); TraceMapperV0_4 traceMapper = new TraceMapperV0_4(); PayloadVerifier verifier = new PayloadVerifier( traces, traceMapper, - (expected, received) -> { + (expectedObj, received) -> { + List expected = (List) expectedObj; MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(received); - List expectedStack = (List) expected; int size = unpacker.unpackArrayHeader(); - assertEquals(expectedStack.size(), size); - for (Object entry : expectedStack) { - @SuppressWarnings("unchecked") - Map stackEntry = (Map) entry; + assertEquals(expected.size(), size); + for (int i = 0; i < size; i++) { + Map stackEntry = (Map) expected.get(i); int fields = unpacker.unpackMapHeader(); - for (int f = 0; f < fields; ++f) { + for (int j = 0; j < fields; j++) { String field = unpacker.unpackString(); assertEquals(stackEntry.get(field), unpacker.unpackString()); } @@ -196,12 +188,13 @@ void metaStructSupport() { } @Test - void processTagsSerialization() { + void testProcessTagsSerialization() { assertNotNull(ProcessTags.getTagsForSerialization()); - List spans = new ArrayList<>(); - for (long spanId = 1; spanId <= 2; ++spanId) { + + List spans = new ArrayList<>(); + for (int spanId = 1; spanId <= 2; spanId++) { spans.add( - new PojoSpan( + new TraceGenerator.PojoSpan( "service", "operation", "resource", @@ -220,7 +213,7 @@ void processTagsSerialization() { "origin")); } - List> traces = Collections.singletonList(spans); + List> traces = Collections.singletonList(spans); TraceMapperV0_4 traceMapper = new TraceMapperV0_4(); PayloadVerifier verifier = new PayloadVerifier(traces, traceMapper); MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(20 << 10, verifier)); @@ -231,24 +224,28 @@ void processTagsSerialization() { verifier.verifyTracesConsumed(); } - private static final class PayloadVerifier implements ByteBufferConsumer { + // --- Inner classes --- - private final List> expectedTraces; - private final TraceMapperV0_4 mapper; - private final MetaStructVerifier metaStructVerifier; - private final PayloadVerifiers.CapturingChannel channel = - new PayloadVerifiers.CapturingChannel(200 << 10); + private interface MetaStructVerifier { + void verify(Object expected, byte[] received) throws IOException; + } + + private static final class PayloadVerifier implements ByteBufferConsumer, WritableByteChannel { + private final List> expectedTraces; + private final TraceMapperV0_4 mapper; + private ByteBuffer captured = ByteBuffer.allocate(200 << 10); + private final MetaStructVerifier metaStructVerifier; private int position = 0; - private PayloadVerifier(List> traces, TraceMapperV0_4 mapper) { + private PayloadVerifier(List> traces, TraceMapperV0_4 mapper) { this(traces, mapper, null); } private PayloadVerifier( - List> traces, + List> traces, TraceMapperV0_4 mapper, - MetaStructVerifier metaStructVerifier) { + MetaStructVerifier metaStructVerifier) { this.expectedTraces = traces; this.mapper = mapper; this.metaStructVerifier = metaStructVerifier; @@ -266,15 +263,16 @@ public void accept(int messageCount, ByteBuffer buffer) { int processTagsCount = 0; try { Payload payload = mapper.newPayload().withBody(messageCount, buffer); - payload.writeTo(channel); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(channel.flipForReading()); + payload.writeTo(this); + captured.flip(); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(captured); int traceCount = unpacker.unpackArrayHeader(); for (int i = 0; i < traceCount; ++i) { - List expectedTrace = expectedTraces.get(position++); + List expectedTrace = expectedTraces.get(position++); int spanCount = unpacker.unpackArrayHeader(); assertEquals(expectedTrace.size(), spanCount); for (int k = 0; k < spanCount; ++k) { - PojoSpan expectedSpan = expectedTrace.get(k); + TraceGenerator.PojoSpan expectedSpan = expectedTrace.get(k); int elementCount = unpacker.unpackMapHeader(); boolean hasMetaStruct = !expectedSpan.getMetaStruct().isEmpty(); assertEquals(hasMetaStruct ? 13 : 12, elementCount); @@ -313,19 +311,43 @@ public void accept(int messageCount, ByteBuffer buffer) { HashMap metrics = new HashMap<>(); for (int j = 0; j < metricsSize; ++j) { String key = unpacker.unpackString(); - Number n = unpackNumber(unpacker, key); + Number metricValue = null; + MessageFormat format = unpacker.getNextFormat(); + switch (format) { + case NEGFIXINT: + case POSFIXINT: + case INT8: + case UINT8: + case INT16: + case UINT16: + case INT32: + case UINT32: + metricValue = unpacker.unpackInt(); + break; + case INT64: + case UINT64: + metricValue = unpacker.unpackLong(); + break; + case FLOAT32: + metricValue = unpacker.unpackFloat(); + break; + case FLOAT64: + metricValue = unpacker.unpackDouble(); + break; + default: + Assertions.fail("Unexpected type in metrics values: " + format); + } if (DD_MEASURED.toString().equals(key)) { - assertTrue( - (n.intValue() == 1 && expectedSpan.isMeasured()) || !expectedSpan.isMeasured()); + assertTrue(metricValue.intValue() == 1 || !expectedSpan.isMeasured()); } else if (DDSpanContext.PRIORITY_SAMPLING_KEY.equals(key)) { // check that priority sampling is only on first and last span if (k == 0 || k == spanCount - 1) { - assertEquals(expectedSpan.samplingPriority(), n.intValue()); + assertEquals(expectedSpan.samplingPriority(), metricValue.intValue()); } else { assertFalse(expectedSpan.hasSamplingPriority()); } } else { - metrics.put(key, n); + metrics.put(key, metricValue); } } for (Map.Entry metric : metrics.entrySet()) { @@ -335,8 +357,8 @@ public void accept(int messageCount, ByteBuffer buffer) { metric.getValue().doubleValue(), 0.001); } else { - // Integer-typed metrics round-trip through msgpack's minimal encoding, so a Long - // tag can come back as an Integer (and vice versa). Compare numerically. + // Groovy compared numerically, Java requires explicit long comparison to avoid + // Long/Integer type mismatch from different msgpack integer encoding widths assertEquals( ((Number) expectedSpan.getTag(metric.getKey())).longValue(), metric.getValue().longValue()); @@ -360,7 +382,7 @@ public void accept(int messageCount, ByteBuffer buffer) { processTagsCount++; } else { Object tag = expectedSpan.getTag(entry.getKey()); - if (tag != null) { + if (null != tag) { assertEquals(String.valueOf(tag), entry.getValue()); } else { assertEquals(expectedSpan.getBaggage().get(entry.getKey()), entry.getValue()); @@ -383,22 +405,47 @@ public void accept(int messageCount, ByteBuffer buffer) { } } } catch (IOException e) { - fail(e.getMessage()); + Assertions.fail(e.getMessage()); } finally { mapper.reset(); - channel.resetForWriting(); + captured.position(0); + captured.limit(captured.capacity()); assertEquals( Config.get().isExperimentalPropagateProcessTagsEnabled() ? 1 : 0, processTagsCount); } } + @Override + public int write(ByteBuffer src) { + if (captured.remaining() < src.remaining()) { + ByteBuffer newBuffer = ByteBuffer.allocate(captured.capacity() + src.capacity()); + captured.flip(); + newBuffer.put(captured); + captured = newBuffer; + return write(src); + } + captured.put(src); + return src.position(); + } + void verifyTracesConsumed() { assertEquals(expectedTraces.size(), position); } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void close() {} } - @FunctionalInterface - private interface MetaStructVerifier { - void verify(E expected, byte[] received) throws IOException; + private static void assertEqualsWithNullAsEmpty(CharSequence expected, CharSequence actual) { + if (null == expected) { + assertEquals("", actual); + } else { + assertEquals(expected.toString(), actual.toString()); + } } } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV05PayloadTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV05PayloadTest.java index 6dedd2e9ef6..453a35f04bc 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV05PayloadTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV05PayloadTest.java @@ -3,14 +3,10 @@ import static datadog.trace.api.config.GeneralConfig.EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.DD_MEASURED; import static datadog.trace.common.writer.TraceGenerator.generateRandomTraces; -import static datadog.trace.common.writer.ddagent.PayloadVerifiers.assertEqualsWithNullAsEmpty; -import static datadog.trace.common.writer.ddagent.PayloadVerifiers.unpackNumber; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.junit.jupiter.params.provider.Arguments.arguments; import datadog.communication.serialization.ByteBufferConsumer; import datadog.communication.serialization.FlushingBuffer; @@ -23,12 +19,13 @@ import datadog.trace.api.sampling.PrioritySampling; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.common.writer.Payload; -import datadog.trace.common.writer.TraceGenerator.PojoSpan; +import datadog.trace.common.writer.TraceGenerator; import datadog.trace.core.DDSpanContext; -import datadog.trace.test.junit.utils.config.WithConfig; -import datadog.trace.test.junit.utils.config.WithConfigExtension; +import datadog.trace.junit.utils.config.WithConfig; +import datadog.trace.test.util.DDJavaSpecification; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -36,49 +33,35 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Stream; -import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; +import org.msgpack.core.MessageFormat; import org.msgpack.core.MessagePack; import org.msgpack.core.MessageUnpacker; +import org.tabletest.junit.TableTest; -@ExtendWith(WithConfigExtension.class) -class TraceMapperV05PayloadTest { +class TraceMapperV05PayloadTest extends DDJavaSpecification { - // Keep the ProcessTags static in sync with the (per-test rebuilt) Config, the way DDSpecification - // did for the original Spock tests. Runs after WithConfigExtension has rebuilt Config. @BeforeEach - void syncProcessTags() { + void resetProcessTags() { + // Sync ProcessTags.enabled with the current Config (which may be modified by @WithConfig) ProcessTags.reset(Config.get()); } - // A method-level @WithConfig below disables process tags, but WithConfigExtension only rebuilds - // Config after each test -- it does not reset the ProcessTags static. Reset it once the class is - // done so a disabled state can't leak into other tests sharing this Gradle JVM. By afterAll the - // extension has already restored Config to its defaults. - @AfterAll - static void resetProcessTags() { - ProcessTags.reset(Config.get()); - } - - // disable process tags since they are only on the first span of the chunk otherwise the - // calculation woes - @Test @WithConfig(key = EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED, value = "false") - void bodyOverflowCausesAFlush() { + @Test + void testBodyOverflowCausesFlush() { + // disable process tags since they are only on the first span of the chunk otherwise the + // calculation woes // 4x 36 ASCII characters and 2 bytes of msgpack string prefix int dictionarySpacePerTrace = 4 * (36 + 2); // enough space for two traces with distinct string values, plus the header int dictionarySize = dictionarySpacePerTrace * 2 + 5; TraceMapperV0_5 traceMapper = new TraceMapperV0_5(dictionarySize); - List repeatedTrace = + List repeatedTrace = Collections.singletonList( - new PojoSpan( + new TraceGenerator.PojoSpan( UUID.randomUUID().toString(), UUID.randomUUID().toString(), UUID.randomUUID().toString(), @@ -99,83 +82,86 @@ void bodyOverflowCausesAFlush() { // 30KB body int bufferSize = 30 << 10; int tracesRequiredToOverflowBody = (int) Math.ceil((double) bufferSize / traceSize) + 1; - List> traces = new ArrayList<>(tracesRequiredToOverflowBody); + List> traces = new ArrayList<>(tracesRequiredToOverflowBody); for (int i = 0; i < tracesRequiredToOverflowBody; ++i) { traces.add(repeatedTrace); } // the last one won't be flushed - List> flushedTraces = new ArrayList<>(traces); + List> flushedTraces = new ArrayList<>(traces); flushedTraces.remove(traces.size() - 1); // need space for the overflowing buffer, the dictionary, and two small array headers PayloadVerifier verifier = new PayloadVerifier(flushedTraces, traceMapper, bufferSize + dictionarySize + 1 + 1 + 5); MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(bufferSize, verifier)); - for (List trace : traces) { + + for (List trace : traces) { packer.format(trace, traceMapper); } + verifier.verifyTracesConsumed(); } - @ParameterizedTest(name = "buffer={0} dict={1} traces={2} lowCardinality={3}") - @MethodSource("dictionaryCompressedTracesWrittenCorrectlyArguments") - void dictionaryCompressedTracesWrittenCorrectly( + @TableTest({ + "scenario | bufferSize | dictionarySize | traceCount | lowCardinality", + "0 traces 10k buf 10k dict low | 10240 | 10240 | 0 | true ", + "1 trace 10k buf 10k dict low | 10240 | 10240 | 1 | true ", + "10 traces 10k buf 10k dict low | 10240 | 10240 | 10 | true ", + "100 traces 10k buf 10k dict low | 10240 | 10240 | 100 | true ", + "1 trace 10k buf 100k dict low | 10240 | 102400 | 1 | true ", + "10 traces 10k buf 100k dict low | 10240 | 102400 | 10 | true ", + "100 traces 10k buf 100k dict low | 10240 | 102400 | 100 | true ", + "0 traces 10k buf 10k dict high | 10240 | 10240 | 0 | false ", + "1 trace 10k buf 10k dict high | 10240 | 10240 | 1 | false ", + "10 traces 10k buf 10k dict high | 10240 | 10240 | 10 | false ", + "100 traces 10k buf 10k dict high | 10240 | 10240 | 100 | false ", + "1 trace 10k buf 100k dict high | 10240 | 102400 | 1 | false ", + "10 traces 10k buf 100k dict high | 10240 | 102400 | 10 | false ", + "100 traces 10k buf 100k dict high | 10240 | 102400 | 100 | false ", + "0 traces 100k buf 10k dict low | 102400 | 10240 | 0 | true ", + "1 trace 100k buf 10k dict low | 102400 | 10240 | 1 | true ", + "10 traces 100k buf 10k dict low | 102400 | 10240 | 10 | true ", + "100 traces 100k buf 10k dict low | 102400 | 10240 | 100 | true ", + "1 trace 100k buf 100k dict low | 102400 | 102400 | 1 | true ", + "10 traces 100k buf 100k dict low | 102400 | 102400 | 10 | true ", + "100 traces 100k buf 100k dict low | 102400 | 102400 | 100 | true ", + "0 traces 100k buf 10k dict high | 102400 | 10240 | 0 | false ", + "1 trace 100k buf 10k dict high | 102400 | 10240 | 1 | false ", + "10 traces 100k buf 10k dict high | 102400 | 10240 | 10 | false ", + "100 traces 100k buf 10k dict high | 102400 | 10240 | 100 | false ", + "1 trace 100k buf 100k dict high | 102400 | 102400 | 1 | false ", + "10 traces 100k buf 100k dict high | 102400 | 102400 | 10 | false ", + "100 traces 100k buf 100k dict high | 102400 | 102400 | 100 | false ", + "1000 traces 100k buf 100k dict high | 102400 | 102400 | 1000 | false " + }) + void testDictionaryCompressedTracesWrittenCorrectly( int bufferSize, int dictionarySize, int traceCount, boolean lowCardinality) { - List> traces = generateRandomTraces(traceCount, lowCardinality); + List> traces = generateRandomTraces(traceCount, lowCardinality); TraceMapperV0_5 traceMapper = new TraceMapperV0_5(dictionarySize); PayloadVerifier verifier = new PayloadVerifier(traces, traceMapper); MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(bufferSize, verifier)); + boolean tracesFitInBuffer = true; - for (List trace : traces) { + for (List trace : traces) { if (!packer.format(trace, traceMapper)) { verifier.skipLargeTrace(); tracesFitInBuffer = false; } } packer.flush(); + if (tracesFitInBuffer) { verifier.verifyTracesConsumed(); } } - private static Stream dictionaryCompressedTracesWrittenCorrectlyArguments() { - return Stream.of( - arguments(10 << 10, 10 << 10, 0, true), - arguments(10 << 10, 10 << 10, 1, true), - arguments(10 << 10, 10 << 10, 10, true), - arguments(10 << 10, 10 << 10, 100, true), - arguments(10 << 10, 100 << 10, 1, true), - arguments(10 << 10, 100 << 10, 10, true), - arguments(10 << 10, 100 << 10, 100, true), - arguments(10 << 10, 10 << 10, 0, false), - arguments(10 << 10, 10 << 10, 1, false), - arguments(10 << 10, 10 << 10, 10, false), - arguments(10 << 10, 10 << 10, 100, false), - arguments(10 << 10, 100 << 10, 1, false), - arguments(10 << 10, 100 << 10, 10, false), - arguments(10 << 10, 100 << 10, 100, false), - arguments(100 << 10, 10 << 10, 0, true), - arguments(100 << 10, 10 << 10, 1, true), - arguments(100 << 10, 10 << 10, 10, true), - arguments(100 << 10, 10 << 10, 100, true), - arguments(100 << 10, 100 << 10, 1, true), - arguments(100 << 10, 100 << 10, 10, true), - arguments(100 << 10, 100 << 10, 100, true), - arguments(100 << 10, 10 << 10, 0, false), - arguments(100 << 10, 10 << 10, 1, false), - arguments(100 << 10, 10 << 10, 10, false), - arguments(100 << 10, 10 << 10, 100, false), - arguments(100 << 10, 100 << 10, 1, false), - arguments(100 << 10, 100 << 10, 10, false), - arguments(100 << 10, 100 << 10, 1000, false)); - } - @Test - void processTagsSerialization() { + void testProcessTagsSerialization() { assertNotNull(ProcessTags.getTagsForSerialization()); - List spans = new ArrayList<>(); - for (long spanId = 1; spanId <= 2; ++spanId) { + + List spans = new ArrayList<>(); + for (int spanId = 1; spanId <= 2; spanId++) { spans.add( - new PojoSpan( + new TraceGenerator.PojoSpan( "service", "operation", "resource", @@ -194,7 +180,7 @@ void processTagsSerialization() { "origin")); } - List> traces = Collections.singletonList(spans); + List> traces = Collections.singletonList(spans); TraceMapperV0_5 traceMapper = new TraceMapperV0_5(); PayloadVerifier verifier = new PayloadVerifier(traces, traceMapper); MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(20 << 10, verifier)); @@ -205,22 +191,24 @@ void processTagsSerialization() { verifier.verifyTracesConsumed(); } - private static final class PayloadVerifier implements ByteBufferConsumer { + // --- Inner classes --- - private final List> expectedTraces; - private final TraceMapperV0_5 mapper; - private final PayloadVerifiers.CapturingChannel channel; + private static final class PayloadVerifier implements ByteBufferConsumer, WritableByteChannel { + private final List> expectedTraces; + private final TraceMapperV0_5 mapper; + private ByteBuffer captured; private int position = 0; - private PayloadVerifier(List> traces, TraceMapperV0_5 mapper) { + private PayloadVerifier(List> traces, TraceMapperV0_5 mapper) { this(traces, mapper, 200 << 10); } - private PayloadVerifier(List> traces, TraceMapperV0_5 mapper, int size) { + private PayloadVerifier( + List> traces, TraceMapperV0_5 mapper, int size) { this.expectedTraces = traces; this.mapper = mapper; - this.channel = new PayloadVerifiers.CapturingChannel(size); + this.captured = ByteBuffer.allocate(size); } void skipLargeTrace() { @@ -232,8 +220,9 @@ public void accept(int messageCount, ByteBuffer buffer) { int processTagsCount = 0; try { Payload payload = mapper.newPayload().withBody(messageCount, buffer); - payload.writeTo(channel); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(channel.flipForReading()); + payload.writeTo(this); + captured.flip(); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(captured); int header = unpacker.unpackArrayHeader(); assertEquals(2, header); int dictionarySize = unpacker.unpackArrayHeader(); @@ -243,11 +232,11 @@ public void accept(int messageCount, ByteBuffer buffer) { } int traceCount = unpacker.unpackArrayHeader(); for (int i = 0; i < traceCount; ++i) { - List expectedTrace = expectedTraces.get(position++); + List expectedTrace = expectedTraces.get(position++); int spanCount = unpacker.unpackArrayHeader(); assertEquals(expectedTrace.size(), spanCount); for (int k = 0; k < spanCount; ++k) { - PojoSpan expectedSpan = expectedTrace.get(k); + TraceGenerator.PojoSpan expectedSpan = expectedTrace.get(k); int elementCount = unpacker.unpackArrayHeader(); assertEquals(12, elementCount); String serviceName = dictionary[unpacker.unpackInt()]; @@ -285,7 +274,7 @@ public void accept(int messageCount, ByteBuffer buffer) { assertEquals(ProcessTags.getTagsForSerialization().toString(), entry.getValue()); } else { Object tag = expectedSpan.getTag(entry.getKey()); - if (tag != null) { + if (null != tag) { assertEquals(String.valueOf(tag), entry.getValue()); } else { assertEquals(expectedSpan.getBaggage().get(entry.getKey()), entry.getValue()); @@ -296,19 +285,44 @@ public void accept(int messageCount, ByteBuffer buffer) { HashMap metrics = new HashMap<>(); for (int j = 0; j < metricsSize; ++j) { String key = dictionary[unpacker.unpackInt()]; - Number n = unpackNumber(unpacker, key); + Number metricValue = null; + MessageFormat format = unpacker.getNextFormat(); + switch (format) { + case NEGFIXINT: + case POSFIXINT: + case INT8: + case UINT8: + case INT16: + case UINT16: + case INT32: + case UINT32: + metricValue = unpacker.unpackInt(); + break; + case INT64: + case UINT64: + metricValue = unpacker.unpackLong(); + break; + case FLOAT32: + metricValue = unpacker.unpackFloat(); + break; + case FLOAT64: + metricValue = unpacker.unpackDouble(); + break; + default: + Assertions.fail( + "Unexpected type in metrics values: " + format + " for key " + key); + } if (DD_MEASURED.toString().equals(key)) { - assertTrue( - (n.intValue() == 1 && expectedSpan.isMeasured()) || !expectedSpan.isMeasured()); + assertTrue(metricValue.intValue() == 1 || !expectedSpan.isMeasured()); } else if (DDSpanContext.PRIORITY_SAMPLING_KEY.equals(key)) { // check that priority sampling is only on first and last span if (k == 0 || k == spanCount - 1) { - assertEquals(expectedSpan.samplingPriority(), n.intValue()); + assertEquals(expectedSpan.samplingPriority(), metricValue.intValue()); } else { assertFalse(expectedSpan.hasSamplingPriority()); } } else { - metrics.put(key, n); + metrics.put(key, metricValue); } } for (Map.Entry metric : metrics.entrySet()) { @@ -319,8 +333,7 @@ public void accept(int messageCount, ByteBuffer buffer) { 0.001, metric.getKey()); } else { - // Integer-typed metrics round-trip through msgpack's minimal encoding, so a Long - // tag can come back as an Integer (and vice versa). Compare numerically. + // Groovy compared numerically; use longValue() to avoid Long/Integer type mismatch assertEquals( ((Number) expectedSpan.getTag(metric.getKey())).longValue(), metric.getValue().longValue(), @@ -332,32 +345,56 @@ public void accept(int messageCount, ByteBuffer buffer) { } } } catch (IOException e) { - fail(e.getMessage()); + Assertions.fail(e.getMessage()); } finally { assertEquals( Config.get().isExperimentalPropagateProcessTagsEnabled() ? 1 : 0, processTagsCount); mapper.reset(); - channel.resetForWriting(); + captured.position(0); + captured.limit(captured.capacity()); } } + @Override + public int write(ByteBuffer src) { + if (captured.remaining() < src.remaining()) { + ByteBuffer newBuffer = ByteBuffer.allocate(captured.capacity() + src.remaining()); + captured.flip(); + newBuffer.put(captured); + captured = newBuffer; + return write(src); + } + captured.put(src); + return src.position(); + } + void verifyTracesConsumed() { assertEquals(expectedTraces.size(), position); } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void close() {} + } + + private static void assertEqualsWithNullAsEmpty(CharSequence expected, CharSequence actual) { + if (null == expected) { + assertEquals("", actual); + } else { + assertEquals(expected.toString(), actual.toString()); + } } - private static int calculateSize(List trace) { + static int calculateSize(List trace) { AtomicInteger size = new AtomicInteger(); MsgPackWriter packer = new MsgPackWriter( new FlushingBuffer( - 1024, - new ByteBufferConsumer() { - @Override - public void accept(int messageCount, ByteBuffer buffer) { - size.set(buffer.limit() - buffer.position()); - } - })); + 1024, (messageCount, buffer) -> size.set(buffer.limit() - buffer.position()))); packer.format(trace, new TraceMapperV0_5(1024)); packer.flush(); return size.get(); diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java index 1eacd903b96..2b5ea0bf7c7 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java @@ -1,151 +1,76 @@ package datadog.trace.common.writer.ddagent; -import static datadog.trace.api.DDTags.PROCESS_TAGS; -import static datadog.trace.api.DDTags.SPAN_EVENTS; -import static datadog.trace.api.DDTags.THREAD_ID; -import static datadog.trace.api.DDTags.THREAD_NAME; -import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; -import static datadog.trace.bootstrap.instrumentation.api.Tags.COMPONENT; -import static datadog.trace.bootstrap.instrumentation.api.Tags.ENV; -import static datadog.trace.bootstrap.instrumentation.api.Tags.HTTP_STATUS; -import static datadog.trace.bootstrap.instrumentation.api.Tags.HTTP_URL; -import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND; -import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND_CLIENT; -import static datadog.trace.bootstrap.instrumentation.api.Tags.VERSION; import static datadog.trace.common.writer.TraceGenerator.generateRandomTraces; -import static datadog.trace.common.writer.ddagent.PayloadVerifiers.assertEqualsWithNullAsEmpty; -import static datadog.trace.common.writer.ddagent.V1PayloadReader.newStringTable; -import static datadog.trace.common.writer.ddagent.V1PayloadReader.readAttributes; -import static datadog.trace.common.writer.ddagent.V1PayloadReader.readBinary; -import static datadog.trace.common.writer.ddagent.V1PayloadReader.readFirstChunk; -import static datadog.trace.common.writer.ddagent.V1PayloadReader.readFirstSpan; -import static datadog.trace.common.writer.ddagent.V1PayloadReader.readStreamingString; -import static datadog.trace.common.writer.ddagent.V1PayloadReader.skipChunkField; -import static datadog.trace.common.writer.ddagent.V1PayloadReader.skipPayloadField; -import static datadog.trace.common.writer.ddagent.V1PayloadReader.skipSpanField; -import static datadog.trace.common.writer.ddagent.V1PayloadReader.traceIdBytes; -import static datadog.trace.common.writer.ddagent.V1PayloadReader.unpackUnsignedLong; -import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; +import static org.msgpack.core.MessageFormat.FIXSTR; +import static org.msgpack.core.MessageFormat.STR16; +import static org.msgpack.core.MessageFormat.STR32; +import static org.msgpack.core.MessageFormat.STR8; import datadog.communication.serialization.ByteBufferConsumer; import datadog.communication.serialization.FlushingBuffer; import datadog.communication.serialization.msgpack.MsgPackWriter; -import datadog.trace.api.Config; import datadog.trace.api.DDSpanId; +import datadog.trace.api.DDTags; import datadog.trace.api.DDTraceId; import datadog.trace.api.ProcessTags; +import datadog.trace.api.sampling.PrioritySampling; import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink; import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags; import datadog.trace.bootstrap.instrumentation.api.SpanAttributes; import datadog.trace.bootstrap.instrumentation.api.SpanLink; +import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.common.writer.Payload; -import datadog.trace.common.writer.TraceGenerator.PojoSpan; -import datadog.trace.common.writer.ddagent.V1PayloadReader.ChunkField; -import datadog.trace.common.writer.ddagent.V1PayloadReader.PayloadField; -import datadog.trace.common.writer.ddagent.V1PayloadReader.SpanField; -import datadog.trace.common.writer.ddagent.V1PayloadReader.V1SpanEvent; -import datadog.trace.common.writer.ddagent.V1PayloadReader.V1SpanLink; +import datadog.trace.common.writer.TraceGenerator; import datadog.trace.core.MetadataConsumer; -import datadog.trace.test.junit.utils.config.WithConfigExtension; -import java.io.ByteArrayOutputStream; +import datadog.trace.junit.utils.tabletest.SamplingMechanismConverter; +import datadog.trace.test.util.DDJavaSpecification; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.converter.ConvertWith; import org.msgpack.core.MessageFormat; import org.msgpack.core.MessagePack; import org.msgpack.core.MessageUnpacker; -import org.msgpack.value.ValueType; import org.tabletest.junit.TableTest; -@ExtendWith(WithConfigExtension.class) -class TraceMapperV1PayloadTest { - - private static final String DECISION_MAKER_TAG = "_dd.p.dm"; - - /** Every field the v1 payload header carries, mirroring {@code TraceMapperV1.buildHeader}. */ - private static final Set EXPECTED_PAYLOAD_FIELD_IDS = - new HashSet<>( - asList( - PayloadField.CONTAINER_ID, - PayloadField.LANGUAGE_NAME, - PayloadField.LANGUAGE_VERSION, - PayloadField.TRACER_VERSION, - PayloadField.RUNTIME_ID, - PayloadField.ENV, - PayloadField.HOSTNAME, - PayloadField.APP_VERSION, - PayloadField.ATTRIBUTES, - PayloadField.CHUNKS)); - - /** Every field a v1 span carries, mirroring {@code TraceMapperV1.encodeSpans}. */ - private static final Set EXPECTED_SPAN_FIELD_IDS = - new HashSet<>( - asList( - SpanField.SERVICE, - SpanField.NAME, - SpanField.RESOURCE, - SpanField.SPAN_ID, - SpanField.PARENT_ID, - SpanField.START, - SpanField.DURATION, - SpanField.ERROR, - SpanField.ATTRIBUTES, - SpanField.TYPE, - SpanField.LINKS, - SpanField.EVENTS, - SpanField.ENV, - SpanField.VERSION, - SpanField.COMPONENT, - SpanField.KIND)); - - // Keep the ProcessTags static in sync with the (per-test rebuilt) Config, the way DDSpecification - // did for the original Spock tests. Runs after WithConfigExtension has rebuilt Config. - @BeforeEach - void syncProcessTags() { - ProcessTags.reset(Config.get()); - } +class TraceMapperV1PayloadTest extends DDJavaSpecification { @TableTest({ - "scenario | bufferSizeKb | traceCount | lowCardinality", - "no traces, low cardinality | 20 | 0 | true ", - "one trace, low cardinality | 20 | 1 | true ", - "two traces, low cardinality | 30 | 2 | true ", - "no traces, high cardinality | 20 | 0 | false ", - "one trace, high cardinality | 20 | 1 | false ", - "two traces, high cardinality | 30 | 2 | false ", - "ten traces, low cardinality | 100 | 10 | true ", - "hundred traces, high cardinality | 100 | 100 | false " + "scenario | bufferSize | traceCount | lowCardinality", + "0 traces low card | 20480 | 0 | true ", + "1 trace low card | 20480 | 1 | true ", + "2 traces low card | 30720 | 2 | true ", + "0 traces high card | 20480 | 0 | false ", + "1 trace high card | 20480 | 1 | false ", + "2 traces high card | 30720 | 2 | false ", + "10 traces low card | 102400 | 10 | true ", + "100 traces high card | 102400 | 100 | false " }) - void tracesWrittenCorrectly( - String scenario, int bufferSizeKb, int traceCount, boolean lowCardinality) { - List> traces = generateRandomTraces(traceCount, lowCardinality); + void testTracesWrittenCorrectly(int bufferSize, int traceCount, boolean lowCardinality) { + List> traces = generateRandomTraces(traceCount, lowCardinality); TraceMapperV1 traceMapper = new TraceMapperV1(); PayloadVerifier verifier = new PayloadVerifier(traces, traceMapper); - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(bufferSizeKb << 10, verifier)); + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(bufferSize, verifier)); boolean tracesFitInBuffer = true; - for (List trace : traces) { + for (List trace : traces) { if (!packer.format(trace, traceMapper)) { verifier.skipLargeTrace(); tracesFitInBuffer = false; @@ -160,37 +85,40 @@ void tracesWrittenCorrectly( } @Test - void endpointReturnsV1() { + void testEndpointReturnsV10() { assertEquals("v1.0", new TraceMapperV1().endpoint()); } - // expectedKind holds the wire values, which must stay in sync with TraceMapperV1.SPAN_KIND_*. - @TableTest({ - "scenario | spanKind | expectedKind", - "no span.kind tag | | 0 ", - "internal span kind | internal | 1 ", - "server span kind | server | 2 ", - "client span kind | client | 3 ", - "producer span kind | producer | 4 ", - "consumer span kind | consumer | 5 ", - "unrecognized span kind | unknown | 1 " - }) - void spanKindValueConversion(String scenario, String spanKind, int expectedKind) { - assertEquals(expectedKind, TraceMapperV1.getSpanKindValue(spanKind)); + @Test + void testSpanKindValueConversion() { + assertEquals(TraceMapperV1.SPAN_KIND_UNSPECIFIED, TraceMapperV1.getSpanKindValue(null)); + assertEquals( + TraceMapperV1.SPAN_KIND_INTERNAL, TraceMapperV1.getSpanKindValue(Tags.SPAN_KIND_INTERNAL)); + assertEquals( + TraceMapperV1.SPAN_KIND_SERVER, TraceMapperV1.getSpanKindValue(Tags.SPAN_KIND_SERVER)); + assertEquals( + TraceMapperV1.SPAN_KIND_CLIENT, TraceMapperV1.getSpanKindValue(Tags.SPAN_KIND_CLIENT)); + assertEquals( + TraceMapperV1.SPAN_KIND_PRODUCER, TraceMapperV1.getSpanKindValue(Tags.SPAN_KIND_PRODUCER)); + assertEquals( + TraceMapperV1.SPAN_KIND_CONSUMER, TraceMapperV1.getSpanKindValue(Tags.SPAN_KIND_CONSUMER)); + assertEquals(TraceMapperV1.SPAN_KIND_INTERNAL, TraceMapperV1.getSpanKindValue("unknown")); } @Test - void payloadContainsExpectedHeaderAndChunkFields() throws IOException { + void testPayloadContainsExpectedHeaderAndChunkFields() throws IOException { Map tags = new HashMap<>(); - tags.put(ENV, "prod"); - tags.put(VERSION, "1.2.3"); - tags.put(COMPONENT, "http-client"); - tags.put(SPAN_KIND, SPAN_KIND_CLIENT); + tags.put(Tags.ENV, "prod"); + tags.put(Tags.VERSION, "1.2.3"); + tags.put(Tags.COMPONENT, "http-client"); + tags.put(Tags.SPAN_KIND, Tags.SPAN_KIND_CLIENT); tags.put("attr.string", "value"); tags.put("attr.bool", true); tags.put("attr.number", 12.5d); - PojoSpan span = - new PojoSpan( + tags.put("_dd.p.dm", "-3"); + + TraceGenerator.PojoSpan span = + new TraceGenerator.PojoSpan( "service-a", "operation-a", "resource-a", @@ -200,115 +128,205 @@ void payloadContainsExpectedHeaderAndChunkFields() throws IOException { 1000L, 2000L, 1, - singletonMap(DECISION_MAKER_TAG, "-3"), + Collections.emptyMap(), tags, "web", false, - SAMPLER_KEEP, + PrioritySampling.SAMPLER_KEEP, 200, "rum"); - byte[] encoded = serializeV1Payload(span); - List stringTable = newStringTable(); - - try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded)) { - int payloadFieldCount = unpacker.unpackMapHeader(); - Set payloadFieldsSeen = new HashSet<>(); - int chunkCount = -1; - Map payloadAttributes = null; - - for (int i = 0; i < payloadFieldCount; i++) { - int fieldId = unpacker.unpackInt(); - payloadFieldsSeen.add(fieldId); - if (fieldId == PayloadField.CHUNKS) { + TraceMapperV1 mapper = new TraceMapperV1(); + byte[] encoded = + serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); + List stringTable = new ArrayList<>(); + stringTable.add(""); + + int payloadFieldCount = unpacker.unpackMapHeader(); + Set payloadFieldsSeen = new HashSet<>(); + int chunkCount = -1; + Map payloadAttributes = null; + + for (int i = 0; i < payloadFieldCount; i++) { + int fieldId = unpacker.unpackInt(); + payloadFieldsSeen.add(fieldId); + switch (fieldId) { + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + case 9: + readStreamingString(unpacker, stringTable); + break; + case 10: + payloadAttributes = readAttributes(unpacker, stringTable); + break; + case 11: chunkCount = unpacker.unpackArrayHeader(); assertEquals(1, chunkCount); - verifyChunk(unpacker, singletonList(span), stringTable); - } else if (fieldId == PayloadField.ATTRIBUTES) { - payloadAttributes = readAttributes(unpacker, stringTable); - } else { - skipPayloadField(unpacker, fieldId, stringTable); - } + verifyChunk(unpacker, Collections.singletonList(span), stringTable); + break; + default: + Assertions.fail("Unexpected payload field id: " + fieldId); } + } - assertEquals(EXPECTED_PAYLOAD_FIELD_IDS.size(), payloadFieldCount); - assertEquals(EXPECTED_PAYLOAD_FIELD_IDS, payloadFieldsSeen); - assertEquals(1, chunkCount); - assertNotNull(payloadAttributes); - CharSequence processTags = ProcessTags.getTagsForSerialization(); - if (processTags == null) { - assertEquals(0, payloadAttributes.size()); - } else { - assertEquals(1, payloadAttributes.size()); - assertEquals(processTags.toString(), payloadAttributes.get(PROCESS_TAGS)); - } + assertEquals(10, payloadFieldCount); + Set expectedFields = new HashSet<>(); + for (int i = 2; i <= 11; i++) { + expectedFields.add(i); + } + assertEquals(expectedFields, payloadFieldsSeen); + assertEquals(1, chunkCount); + assertNotNull(payloadAttributes); + if (ProcessTags.getTagsForSerialization() == null) { + assertEquals(0, payloadAttributes.size()); + } else { + assertEquals(1, payloadAttributes.size()); + assertEquals( + ProcessTags.getTagsForSerialization().toString(), + payloadAttributes.get(DDTags.PROCESS_TAGS)); } } - // expectedSamplingMechanism 0 is SamplingMechanism.DEFAULT. @TableTest({ - "scenario | decisionMakerTag | expectedSamplingMechanism", - "no _dd.p.dm tag | | 0 ", - "negative numeric value | '-3' | 3 ", - "hashed prefix before the dash | '934086a686-7' | 7 ", - "unparseable value | invalid | 0 " + "scenario | decisionMakerTag | expectedSamplingMechanism", + "null tag | | SamplingMechanism.DEFAULT", + "simple negative | '-3' | 3 ", + "compound | '934086a686-7' | 7 ", + "invalid | 'invalid' | SamplingMechanism.DEFAULT" }) - void samplingMechanismNormalizationFromDecisionMaker( - String scenario, String decisionMakerTag, int expectedSamplingMechanism) throws IOException { - Map baggage = new HashMap<>(); - if (decisionMakerTag != null) { - baggage.put(DECISION_MAKER_TAG, decisionMakerTag); - } + void testSamplingMechanismNormalizationFromDdPDm( + String decisionMakerTag, + @ConvertWith(SamplingMechanismConverter.class) int expectedSamplingMechanism) + throws IOException { + Map dmTags = + decisionMakerTag == null + ? Collections.emptyMap() + : Collections.singletonMap("_dd.p.dm", decisionMakerTag); - byte[] encoded = serializeV1Payload(spanWithBaggage(baggage)); + TraceGenerator.PojoSpan span = + new TraceGenerator.PojoSpan( + "service-a", + "operation-a", + "resource-a", + DDTraceId.ONE, + 321L, + 0L, + 1000L, + 2000L, + 0, + Collections.emptyMap(), + dmTags, + "custom", + false, + PrioritySampling.SAMPLER_KEEP, + 200, + null); + + TraceMapperV1 mapper = new TraceMapperV1(); + byte[] encoded = + serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); + List stringTable = new ArrayList<>(); + stringTable.add(""); + + unpacker.unpackMapHeader(); + int samplingMechanism = -1; + + for (int i = 0; i < 10; i++) { + int payloadFieldId = unpacker.unpackInt(); + if (payloadFieldId == 11) { + int chunkCount = unpacker.unpackArrayHeader(); + assertEquals(1, chunkCount); + int chunkFieldCount = unpacker.unpackMapHeader(); + for (int j = 0; j < chunkFieldCount; j++) { + int chunkFieldId = unpacker.unpackInt(); + if (chunkFieldId == 7) { + samplingMechanism = unpacker.unpackInt(); + } else { + skipChunkField(unpacker, chunkFieldId, stringTable); + } + } + } else { + skipPayloadField(unpacker, payloadFieldId, stringTable); + } + } - assertEquals(expectedSamplingMechanism, readFirstChunk(encoded).getSamplingMechanism()); + assertEquals(expectedSamplingMechanism, samplingMechanism); } @Test - void spanIdsAreEncodedAsUnsignedValues() throws IOException { + void testSpanIdsAreEncodedAsUnsignedValuesInV1Payloads() throws IOException { long spanId = Long.MIN_VALUE + 123L; long parentId = Long.MIN_VALUE + 456L; + TraceGenerator.PojoSpan span = + new TraceGenerator.PojoSpan( + "service-a", + "operation-a", + "resource-a", + DDTraceId.ONE, + spanId, + parentId, + 1000L, + 2000L, + 0, + Collections.emptyMap(), + Collections.emptyMap(), + "web", + false, + PrioritySampling.SAMPLER_KEEP, + 200, + null); - byte[] encoded = serializeV1Payload(spanWithIds(spanId, parentId)); - List stringTable = newStringTable(); + TraceMapperV1 mapper = new TraceMapperV1(); + byte[] encoded = + serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); + List stringTable = new ArrayList<>(); + stringTable.add(""); + unpacker.unpackMapHeader(); Long actualSpanId = null; Long actualParentId = null; - try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded)) { - int payloadFieldCount = unpacker.unpackMapHeader(); - for (int i = 0; i < payloadFieldCount; i++) { - int payloadFieldId = unpacker.unpackInt(); - if (payloadFieldId != PayloadField.CHUNKS) { - skipPayloadField(unpacker, payloadFieldId, stringTable); - continue; - } - assertEquals(1, unpacker.unpackArrayHeader()); + for (int i = 0; i < 10; i++) { + int payloadFieldId = unpacker.unpackInt(); + if (payloadFieldId == 11) { + int chunkCount = unpacker.unpackArrayHeader(); + assertEquals(1, chunkCount); int chunkFieldCount = unpacker.unpackMapHeader(); for (int j = 0; j < chunkFieldCount; j++) { int chunkFieldId = unpacker.unpackInt(); - if (chunkFieldId != ChunkField.SPANS) { - skipChunkField(unpacker, chunkFieldId, stringTable); - continue; - } - assertEquals(1, unpacker.unpackArrayHeader()); - - int spanFieldCount = unpacker.unpackMapHeader(); - for (int k = 0; k < spanFieldCount; k++) { - int spanFieldId = unpacker.unpackInt(); - switch (spanFieldId) { - case SpanField.SPAN_ID: - actualSpanId = unpackUint64(unpacker); - break; - case SpanField.PARENT_ID: - actualParentId = unpackUint64(unpacker); - break; - default: - skipSpanField(unpacker, spanFieldId, stringTable); + if (chunkFieldId == 4) { + int spanCount = unpacker.unpackArrayHeader(); + assertEquals(1, spanCount); + int spanFieldCount = unpacker.unpackMapHeader(); + for (int k = 0; k < spanFieldCount; k++) { + int spanFieldId = unpacker.unpackInt(); + switch (spanFieldId) { + case 4: + assertEquals(MessageFormat.UINT64, unpacker.getNextFormat()); + actualSpanId = DDSpanId.from(unpacker.unpackBigInteger().toString()); + break; + case 5: + assertEquals(MessageFormat.UINT64, unpacker.getNextFormat()); + actualParentId = DDSpanId.from(unpacker.unpackBigInteger().toString()); + break; + default: + skipSpanField(unpacker, spanFieldId, stringTable); + } } + } else { + skipChunkField(unpacker, chunkFieldId, stringTable); } } + } else { + skipPayloadField(unpacker, payloadFieldId, stringTable); } } @@ -317,183 +335,394 @@ void spanIdsAreEncodedAsUnsignedValues() throws IOException { } @Test - void spanLinksAreEncodedFromStructuredSpanLinks() throws IOException { - DDTraceId firstLinkTraceId = DDTraceId.fromHex("11223344556677889900aabbccddeeff"); - long firstLinkSpanId = DDSpanId.fromHex("000000000000002a"); - Map firstLinkAttributes = new HashMap<>(); - firstLinkAttributes.put("link.kind", "follows_from"); - firstLinkAttributes.put("context_headers", "tracecontext"); - DDTraceId secondLinkTraceId = DDTraceId.fromHex("00000000000000000000000000000001"); - long secondLinkSpanId = DDSpanId.fromHex("0000000000000002"); - List spanLinks = new ArrayList<>(); - spanLinks.add( - new TestSpanLink( - firstLinkTraceId, - firstLinkSpanId, - (byte) 1, - "dd=s:1", - SpanAttributes.fromMap(firstLinkAttributes))); - spanLinks.add( - new TestSpanLink(secondLinkTraceId, secondLinkSpanId, (byte) 0, "", SpanAttributes.EMPTY)); - - List links = readFirstSpan(serializeV1Payload(spanWithLinks(spanLinks))).getLinks(); + void testSpanLinksAreEncodedFromStructuredSpanLinks() throws IOException { + Map linkAttrs = new HashMap<>(); + linkAttrs.put("link.kind", "follows_from"); + linkAttrs.put("context_headers", "tracecontext"); + + List spanLinks = + Arrays.asList( + new TestSpanLink( + DDTraceId.fromHex("11223344556677889900aabbccddeeff"), + DDSpanId.fromHex("000000000000002a"), + (byte) 1, + "dd=s:1", + SpanAttributes.fromMap(linkAttrs)), + new TestSpanLink( + DDTraceId.fromHex("00000000000000000000000000000001"), + DDSpanId.fromHex("0000000000000002"), + (byte) 0, + "", + SpanAttributes.EMPTY)); + + TraceGenerator.PojoSpan span = + new TraceGenerator.PojoSpan( + "service-a", + "operation-a", + "resource-a", + DDTraceId.ONE, + 123L, + 0L, + 1000L, + 2000L, + 0, + Collections.emptyMap(), + Collections.emptyMap(), + "web", + false, + PrioritySampling.SAMPLER_KEEP, + 200, + null, + spanLinks); - assertEquals(2, links.size()); - V1SpanLink firstLink = links.get(0); - assertArrayEquals(traceIdBytes(firstLinkTraceId), firstLink.getTraceId()); - assertEquals(firstLinkSpanId, firstLink.getSpanId()); - assertEquals("dd=s:1", firstLink.getTraceState()); - assertEquals(1L, firstLink.getTraceFlags()); - assertEquals(firstLinkAttributes, firstLink.getAttributes()); + TraceMapperV1 mapper = new TraceMapperV1(); + byte[] encoded = + serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); + List stringTable = new ArrayList<>(); + stringTable.add(""); - V1SpanLink secondLink = links.get(1); - assertArrayEquals(traceIdBytes(secondLinkTraceId), secondLink.getTraceId()); - assertEquals(secondLinkSpanId, secondLink.getSpanId()); - assertEquals("", secondLink.getTraceState()); - assertEquals(0L, secondLink.getTraceFlags()); - assertEquals(emptyMap(), secondLink.getAttributes()); + List> links = readFirstSpanLinks(unpacker, stringTable); + + assertEquals(2, links.size()); + assertArrayEquals( + traceIdBytes(DDTraceId.fromHex("11223344556677889900aabbccddeeff")), + (byte[]) links.get(0).get("traceId")); + assertEquals(DDSpanId.fromHex("000000000000002a"), links.get(0).get("spanId")); + assertEquals("dd=s:1", links.get(0).get("tracestate")); + assertEquals(1L, links.get(0).get("flags")); + Map expectedLinkAttrs0 = new HashMap<>(); + expectedLinkAttrs0.put("link.kind", "follows_from"); + expectedLinkAttrs0.put("context_headers", "tracecontext"); + assertEquals(expectedLinkAttrs0, links.get(0).get("attributes")); + + assertArrayEquals( + traceIdBytes(DDTraceId.fromHex("00000000000000000000000000000001")), + (byte[]) links.get(1).get("traceId")); + assertEquals(DDSpanId.fromHex("0000000000000002"), links.get(1).get("spanId")); + assertEquals("", links.get(1).get("tracestate")); + assertEquals(0L, links.get(1).get("flags")); + assertEquals(Collections.emptyMap(), links.get(1).get("attributes")); } @Test - void firstSpanTagsAreProcessedOnce() { - CountingPojoSpan firstSpan = new CountingPojoSpan("operation-a", "resource-a", 123L, 0L); - CountingPojoSpan secondSpan = new CountingPojoSpan("operation-b", "resource-b", 456L, 123L); - List trace = new ArrayList<>(); - trace.add(firstSpan); - trace.add(secondSpan); + void testFirstSpanTagsAreProcessedOnce() { + CountingPojoSpan firstSpan = + new CountingPojoSpan( + "service-a", + "operation-a", + "resource-a", + DDTraceId.ONE, + 123L, + 0L, + 1000L, + 2000L, + 0, + Collections.emptyMap(), + Collections.singletonMap(Tags.HTTP_URL, "http://localhost:7777/"), + "web", + false, + PrioritySampling.SAMPLER_KEEP, + 200, + null); - serializeV1Payload(trace); + CountingPojoSpan secondSpan = + new CountingPojoSpan( + "service-a", + "operation-b", + "resource-b", + DDTraceId.ONE, + 456L, + 123L, + 1000L, + 2000L, + 0, + Collections.emptyMap(), + Collections.singletonMap(Tags.HTTP_URL, "http://localhost:7777/"), + "web", + false, + PrioritySampling.SAMPLER_KEEP, + 200, + null); + + TraceMapperV1 mapper = new TraceMapperV1(); + + serializeMappedPayload(mapper, Collections.singletonList(Arrays.asList(firstSpan, secondSpan))); assertEquals(1, firstSpan.processTagsAndBaggageCount); assertEquals(1, secondSpan.processTagsAndBaggageCount); } @Test - void missingSpanLinksEncodeEmptyLinks() throws IOException { - byte[] encoded = serializeV1Payload(span(emptyMap())); + void testMissingSpanLinksEncodeEmptyLinks() throws IOException { + TraceGenerator.PojoSpan span = + new TraceGenerator.PojoSpan( + "service-a", + "operation-a", + "resource-a", + DDTraceId.ONE, + 123L, + 0L, + 1000L, + 2000L, + 0, + Collections.emptyMap(), + Collections.emptyMap(), + "web", + false, + PrioritySampling.SAMPLER_KEEP, + 200, + null); + + TraceMapperV1 mapper = new TraceMapperV1(); + byte[] encoded = + serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); + List stringTable = new ArrayList<>(); + stringTable.add(""); - assertTrue(readFirstSpan(encoded).getLinks().isEmpty()); + List> links = readFirstSpanLinks(unpacker, stringTable); + + assertTrue(links.isEmpty()); } @Test - void spanEventsAreEncodedFromEventsTag() throws IOException { - Map firstEventAttributes = new HashMap<>(); - firstEventAttributes.put("str", "v"); - firstEventAttributes.put("int", 42L); - firstEventAttributes.put("double", 12.5d); - firstEventAttributes.put("bool", true); - firstEventAttributes.put("arr", asList("x", 7L, 2.5d, false)); - Map firstEvent = new HashMap<>(); - firstEvent.put("time_unix_nano", 1234567890L); - firstEvent.put("name", "event.one"); - firstEvent.put("attributes", firstEventAttributes); - Map secondEvent = new HashMap<>(); - secondEvent.put("time_unix_nano", 1234567891L); - secondEvent.put("name", "event.two"); - PojoSpan span = span(singletonMap(SPAN_EVENTS, asList(firstEvent, secondEvent))); - - List events = readFirstSpan(serializeV1Payload(span)).getEvents(); + void testSpanEventsAreEncodedFromEventsTag() throws IOException { + Map eventOneAttrs = new HashMap<>(); + eventOneAttrs.put("str", "v"); + eventOneAttrs.put("int", 42L); + eventOneAttrs.put("double", 12.5d); + eventOneAttrs.put("bool", true); + eventOneAttrs.put("arr", Arrays.asList("x", 7L, 2.5d, false)); + + Map eventOne = new HashMap<>(); + eventOne.put("time_unix_nano", 1234567890L); + eventOne.put("name", "event.one"); + eventOne.put("attributes", eventOneAttrs); + + Map eventTwo = new HashMap<>(); + eventTwo.put("time_unix_nano", 1234567891L); + eventTwo.put("name", "event.two"); + + Map tags = + Collections.singletonMap("events", Arrays.asList(eventOne, eventTwo)); + + TraceGenerator.PojoSpan span = + new TraceGenerator.PojoSpan( + "service-a", + "operation-a", + "resource-a", + DDTraceId.ONE, + 123L, + 0L, + 1000L, + 2000L, + 0, + Collections.emptyMap(), + tags, + "web", + false, + PrioritySampling.SAMPLER_KEEP, + 200, + null); + + TraceMapperV1 mapper = new TraceMapperV1(); + byte[] encoded = + serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); + List stringTable = new ArrayList<>(); + stringTable.add(""); + + List> events = readFirstSpanEvents(unpacker, stringTable); assertEquals(2, events.size()); - V1SpanEvent firstDecodedEvent = events.get(0); - assertEquals(1234567890L, firstDecodedEvent.getTimeUnixNano()); - assertEquals("event.one", firstDecodedEvent.getName()); - Map firstDecodedAttributes = firstDecodedEvent.getAttributes(); - assertEquals("v", firstDecodedAttributes.get("str")); - assertEquals(42L, firstDecodedAttributes.get("int")); - assertAttributeValueEquals(12.5d, firstDecodedAttributes.get("double"), "double"); - assertEquals(true, firstDecodedAttributes.get("bool")); - assertEquals(asList("x", 7L, 2.5d, false), firstDecodedAttributes.get("arr")); - - V1SpanEvent secondDecodedEvent = events.get(1); - assertEquals(1234567891L, secondDecodedEvent.getTimeUnixNano()); - assertEquals("event.two", secondDecodedEvent.getName()); - assertEquals(emptyMap(), secondDecodedEvent.getAttributes()); + assertEquals(1234567890L, events.get(0).get("timeUnixNano")); + assertEquals("event.one", events.get(0).get("name")); + Map decodedAttrs0 = (Map) events.get(0).get("attributes"); + assertEquals("v", decodedAttrs0.get("str")); + assertEquals(42L, decodedAttrs0.get("int")); + assertEquals(12.5d, ((Number) decodedAttrs0.get("double")).doubleValue(), 0.000001d); + assertEquals(true, decodedAttrs0.get("bool")); + assertEquals(Arrays.asList("x", 7L, 2.5d, false), decodedAttrs0.get("arr")); + + assertEquals(1234567891L, events.get(1).get("timeUnixNano")); + assertEquals("event.two", events.get(1).get("name")); + assertEquals(Collections.emptyMap(), events.get(1).get("attributes")); } @Test - void malformedSpanEventsFallBackToEmptyEvents() throws IOException { - // a map instead of the expected list of events - PojoSpan span = span(singletonMap(SPAN_EVENTS, singletonMap("foo", "bar"))); + void testMalformedSpanEventsFallBackToEmptyEvents() throws IOException { + Map malformedEvent = Collections.singletonMap("foo", "bar"); + Map tags = Collections.singletonMap("events", malformedEvent); + + TraceGenerator.PojoSpan span = + new TraceGenerator.PojoSpan( + "service-a", + "operation-a", + "resource-a", + DDTraceId.ONE, + 123L, + 0L, + 1000L, + 2000L, + 0, + Collections.emptyMap(), + tags, + "web", + false, + PrioritySampling.SAMPLER_KEEP, + 200, + null); - assertTrue(readFirstSpan(serializeV1Payload(span)).getEvents().isEmpty()); + TraceMapperV1 mapper = new TraceMapperV1(); + byte[] encoded = + serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); + List stringTable = new ArrayList<>(); + stringTable.add(""); + + List> events = readFirstSpanEvents(unpacker, stringTable); + + assertTrue(events.isEmpty()); } @Test - void metaStructIsEncodedAsBytesAttribute() throws IOException { - PojoSpan span = span(emptyMap()); + void testMetaStructIsEncodedAsBytesAttribute() throws IOException { + TraceGenerator.PojoSpan span = + new TraceGenerator.PojoSpan( + "service-a", + "operation-a", + "resource-a", + DDTraceId.ONE, + 123L, + 0L, + 1000L, + 2000L, + 0, + Collections.emptyMap(), + Collections.emptyMap(), + "web", + false, + PrioritySampling.SAMPLER_KEEP, + 200, + null); + Map metaStructValue = new HashMap<>(); metaStructValue.put("foo", "bar"); metaStructValue.put("answer", 42L); span.setMetaStruct("meta_key", metaStructValue); - Map attributes = readFirstSpan(serializeV1Payload(span)).getAttributes(); - byte[] metaStructBytes = (byte[]) attributes.get("meta_key"); - assertNotNull(metaStructBytes); + TraceMapperV1 mapper = new TraceMapperV1(); + byte[] encoded = + serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); + List stringTable = new ArrayList<>(); + stringTable.add(""); + Map attributes = readFirstSpanAttributes(unpacker, stringTable); + byte[] metaStructBytes = (byte[]) attributes.get("meta_key"); + MessageUnpacker metaStructUnpacker = MessagePack.newDefaultUnpacker(metaStructBytes); + int metaStructFieldCount = metaStructUnpacker.unpackMapHeader(); Map decodedMetaStruct = new HashMap<>(); - try (MessageUnpacker metaStructUnpacker = MessagePack.newDefaultUnpacker(metaStructBytes)) { - int metaStructFieldCount = metaStructUnpacker.unpackMapHeader(); - for (int i = 0; i < metaStructFieldCount; i++) { - String key = metaStructUnpacker.unpackString(); - ValueType valueType = metaStructUnpacker.getNextFormat().getValueType(); - switch (valueType) { - case INTEGER: - decodedMetaStruct.put(key, metaStructUnpacker.unpackLong()); - break; - case STRING: - decodedMetaStruct.put(key, metaStructUnpacker.unpackString()); - break; - default: - fail("Unexpected meta_struct value type for key " + key); - } + for (int i = 0; i < metaStructFieldCount; i++) { + String key = metaStructUnpacker.unpackString(); + switch (metaStructUnpacker.getNextFormat().getValueType()) { + case INTEGER: + decodedMetaStruct.put(key, metaStructUnpacker.unpackLong()); + break; + case STRING: + decodedMetaStruct.put(key, metaStructUnpacker.unpackString()); + break; + default: + Assertions.fail("Unexpected meta_struct value type for key " + key); } } + assertNotNull(metaStructBytes); assertEquals("bar", decodedMetaStruct.get("foo")); assertEquals(42L, decodedMetaStruct.get("answer")); } @Test - void mapValuedSpanTagsAreFlattenedInAttributes() throws IOException { - Map user = new HashMap<>(); - user.put("id", "123"); - user.put("name", "alice"); - user.put("authenticated", true); - user.put("profile", singletonMap("age", 30L)); - Map loginMetadata = new HashMap<>(); - loginMetadata.put("event", "login"); - loginMetadata.put("attempts", 1L); - Map loginSuccess = new HashMap<>(); - loginSuccess.put("metadata0", loginMetadata); - loginSuccess.put("metadata1", singletonMap("blocked", false)); + void testMapValuedSpanTagsAreFlattenedInV1Attributes() throws IOException { + Map profile = new HashMap<>(); + profile.put("age", 30L); + + Map usr = new HashMap<>(); + usr.put("id", "123"); + usr.put("name", "alice"); + usr.put("authenticated", true); + usr.put("profile", profile); + + Map metadata0 = new HashMap<>(); + metadata0.put("event", "login"); + metadata0.put("attempts", 1L); + + Map metadata1 = new HashMap<>(); + metadata1.put("blocked", false); + + Map appsecEvents = new HashMap<>(); + appsecEvents.put("metadata0", metadata0); + appsecEvents.put("metadata1", metadata1); + Map tags = new HashMap<>(); - tags.put("usr", user); - String loginSuccessTag = "appsec.events.users.login.success"; - tags.put(loginSuccessTag, loginSuccess); + tags.put("usr", usr); + tags.put("appsec.events.users.login.success", appsecEvents); - // status code 0 keeps the encoder from adding an http.status_code attribute - Map attributes = - readFirstSpan(serializeV1Payload(span(tags, 0))).getAttributes(); + TraceGenerator.PojoSpan span = + new TraceGenerator.PojoSpan( + "service-a", + "operation-a", + "resource-a", + DDTraceId.ONE, + 123L, + 0L, + 1000L, + 2000L, + 0, + Collections.emptyMap(), + tags, + "web", + false, + PrioritySampling.SAMPLER_KEEP, + 0, + null); + + TraceMapperV1 mapper = new TraceMapperV1(); + byte[] encoded = + serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); + List stringTable = new ArrayList<>(); + stringTable.add(""); + + Map attributes = readFirstSpanAttributes(unpacker, stringTable); + + assertTrue(attributes.containsKey("usr.id")); + assertTrue(attributes.containsKey("usr.name")); + assertTrue(attributes.containsKey("usr.authenticated")); + assertTrue(attributes.containsKey("usr.profile.age")); + assertTrue(attributes.containsKey("appsec.events.users.login.success.metadata0.event")); + assertTrue(attributes.containsKey("appsec.events.users.login.success.metadata0.attempts")); + assertTrue(attributes.containsKey("appsec.events.users.login.success.metadata1.blocked")); assertEquals("123", attributes.get("usr.id")); assertEquals("alice", attributes.get("usr.name")); assertEquals(true, attributes.get("usr.authenticated")); - assertAttributeValueEquals(30L, attributes.get("usr.profile.age"), "usr.profile.age"); - assertEquals("login", attributes.get(loginSuccessTag + ".metadata0.event")); - assertAttributeValueEquals( - 1L, attributes.get(loginSuccessTag + ".metadata0.attempts"), "attempts"); - assertEquals(false, attributes.get(loginSuccessTag + ".metadata1.blocked")); - // the 7 flattened entries plus thread.id and thread.name, and nothing else - assertEquals(9, attributes.size()); + assertEquals(30d, ((Number) attributes.get("usr.profile.age")).doubleValue(), 0.000001d); + assertEquals("login", attributes.get("appsec.events.users.login.success.metadata0.event")); + assertEquals( + 1d, + ((Number) attributes.get("appsec.events.users.login.success.metadata0.attempts")) + .doubleValue(), + 0.000001d); + assertEquals(false, attributes.get("appsec.events.users.login.success.metadata1.blocked")); - // the map-valued tags themselves are replaced by their flattened entries assertFalse(attributes.containsKey("usr")); - assertFalse(attributes.containsKey(loginSuccessTag)); + assertFalse(attributes.containsKey("appsec.events.users.login.success")); } @Test - void primitiveSpanTagsAreEncodedInAttributes() throws IOException { + void testPrimitiveSpanTagsAreEncodedInV1Attributes() throws IOException { Map tags = new HashMap<>(); tags.put("tag.bool", true); tags.put("tag.int", 7); @@ -501,43 +730,87 @@ void primitiveSpanTagsAreEncodedInAttributes() throws IOException { tags.put("tag.float", 3.5f); tags.put("tag.double", 4.25d); - // status code 0 keeps the encoder from adding an http.status_code attribute - Map attributes = - readFirstSpan(serializeV1Payload(span(tags, 0))).getAttributes(); + TraceGenerator.PojoSpan span = + new TraceGenerator.PojoSpan( + "service-a", + "operation-a", + "resource-a", + DDTraceId.ONE, + 123L, + 0L, + 1000L, + 2000L, + 0, + Collections.emptyMap(), + tags, + "web", + false, + PrioritySampling.SAMPLER_KEEP, + 0, + null); + + TraceMapperV1 mapper = new TraceMapperV1(); + byte[] encoded = + serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); + List stringTable = new ArrayList<>(); + stringTable.add(""); + + Map attributes = readFirstSpanAttributes(unpacker, stringTable); assertEquals(true, attributes.get("tag.bool")); - assertAttributeValueEquals(7, attributes.get("tag.int"), "tag.int"); - assertAttributeValueEquals(9L, attributes.get("tag.long"), "tag.long"); - assertAttributeValueEquals(3.5f, attributes.get("tag.float"), "tag.float"); - assertAttributeValueEquals(4.25d, attributes.get("tag.double"), "tag.double"); - // the 5 tags plus thread.id and thread.name, and nothing else - assertEquals(7, attributes.size()); + assertEquals(7d, ((Number) attributes.get("tag.int")).doubleValue(), 0.000001d); + assertEquals(9d, ((Number) attributes.get("tag.long")).doubleValue(), 0.000001d); + assertEquals(3.5d, ((Number) attributes.get("tag.float")).doubleValue(), 0.000001d); + assertEquals(4.25d, ((Number) attributes.get("tag.double")).doubleValue(), 0.000001d); } @Test - void threadMetadataIsEncodedInAttributes() throws IOException { - // status code 0 keeps the encoder from adding an http.status_code attribute - PojoSpan span = span(emptyMap(), 0); + void testThreadMetadataIsEncodedInV1Attributes() throws IOException { + TraceGenerator.PojoSpan span = + new TraceGenerator.PojoSpan( + "service-a", + "operation-a", + "resource-a", + DDTraceId.ONE, + 123L, + 0L, + 1000L, + 2000L, + 0, + Collections.emptyMap(), + Collections.emptyMap(), + "web", + false, + PrioritySampling.SAMPLER_KEEP, + 0, + null); - Map attributes = readFirstSpan(serializeV1Payload(span)).getAttributes(); + TraceMapperV1 mapper = new TraceMapperV1(); + byte[] encoded = + serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); + List stringTable = new ArrayList<>(); + stringTable.add(""); + + Map attributes = readFirstSpanAttributes(unpacker, stringTable); - assertAttributeValueEquals(span.getTag(THREAD_ID), attributes.get(THREAD_ID), THREAD_ID); - Object expectedThreadName = span.getTag(THREAD_NAME); - assertEquals(expectedThreadName.toString(), attributes.get(THREAD_NAME)); - // thread metadata is all a tagless span encodes - assertEquals(2, attributes.size()); + assertAttributeValueEquals( + span.getTag(DDTags.THREAD_ID), attributes.get(DDTags.THREAD_ID), DDTags.THREAD_ID); + assertEquals(span.getTag(DDTags.THREAD_NAME).toString(), attributes.get(DDTags.THREAD_NAME)); } - private static final class PayloadVerifier implements ByteBufferConsumer { + // --- Inner classes --- - private final List> expectedTraces; - private final TraceMapperV1 mapper; - private final PayloadVerifiers.CapturingChannel channel = - new PayloadVerifiers.CapturingChannel(200 << 10); + private static final class PayloadVerifier implements ByteBufferConsumer, WritableByteChannel { + private final List> expectedTraces; + private final TraceMapperV1 mapper; + private ByteBuffer captured = ByteBuffer.allocate(200 << 10); private int position = 0; - private PayloadVerifier(List> expectedTraces, TraceMapperV1 mapper) { + private PayloadVerifier( + List> expectedTraces, TraceMapperV1 mapper) { this.expectedTraces = expectedTraces; this.mapper = mapper; } @@ -553,49 +826,188 @@ public void accept(int messageCount, ByteBuffer buffer) { } try { Payload payload = mapper.newPayload().withBody(messageCount, buffer); - payload.writeTo(channel); - try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(channel.flipForReading())) { - if (messageCount == 0) { - assertEquals(0, unpacker.unpackMapHeader()); - return; - } + payload.writeTo(this); + captured.flip(); - List stringTable = newStringTable(); - int payloadFieldCount = unpacker.unpackMapHeader(); - assertEquals(EXPECTED_PAYLOAD_FIELD_IDS.size(), payloadFieldCount); - - boolean seenChunks = false; - for (int i = 0; i < payloadFieldCount; i++) { - int fieldId = unpacker.unpackInt(); - if (fieldId == PayloadField.CHUNKS) { - int traceCount = unpacker.unpackArrayHeader(); - assertEquals(messageCount, traceCount); - seenChunks = true; - for (int traceIndex = 0; traceIndex < traceCount; traceIndex++) { - verifyChunk(unpacker, expectedTraces.get(position++), stringTable); - } - } else { - skipPayloadField(unpacker, fieldId, stringTable); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(captured); + if (messageCount == 0) { + assertEquals(0, unpacker.unpackMapHeader()); + return; + } + + List stringTable = new ArrayList<>(); + stringTable.add(""); + + int payloadFieldCount = unpacker.unpackMapHeader(); + assertEquals(10, payloadFieldCount); + + boolean seenChunks = false; + for (int i = 0; i < payloadFieldCount; i++) { + int fieldId = unpacker.unpackInt(); + if (fieldId == 11) { + int traceCount = unpacker.unpackArrayHeader(); + assertEquals(messageCount, traceCount); + seenChunks = true; + for (int traceIndex = 0; traceIndex < traceCount; traceIndex++) { + List expectedTrace = expectedTraces.get(position++); + verifyChunk(unpacker, expectedTrace, stringTable); } + } else { + skipPayloadField(unpacker, fieldId, stringTable); } - - assertTrue(seenChunks); } + + assertTrue(seenChunks); } catch (IOException e) { - fail(e.getMessage()); + Assertions.fail(e.getMessage()); } finally { mapper.reset(); - channel.resetForWriting(); + captured.position(0); + captured.limit(captured.capacity()); + } + } + + @Override + public int write(ByteBuffer src) { + if (captured.remaining() < src.remaining()) { + ByteBuffer newBuffer = ByteBuffer.allocate(captured.capacity() + src.remaining()); + captured.flip(); + newBuffer.put(captured); + captured = newBuffer; + return write(src); } + captured.put(src); + return src.position(); } void verifyTracesConsumed() { assertEquals(expectedTraces.size(), position); } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void close() {} } + private static class CountingPojoSpan extends TraceGenerator.PojoSpan { + int processTagsAndBaggageCount = 0; + + CountingPojoSpan( + String serviceName, + String operationName, + CharSequence resourceName, + DDTraceId traceId, + long spanId, + long parentId, + long start, + long duration, + int error, + Map baggage, + Map tags, + CharSequence type, + boolean measured, + int samplingPriority, + int statusCode, + CharSequence origin) { + super( + serviceName, + operationName, + resourceName, + traceId, + spanId, + parentId, + start, + duration, + error, + baggage, + tags, + type, + measured, + samplingPriority, + statusCode, + origin); + } + + @Override + public void processTagsAndBaggage(MetadataConsumer consumer) { + processTagsAndBaggageCount++; + super.processTagsAndBaggage(consumer); + } + + @Override + public void processTagsAndBaggage( + MetadataConsumer consumer, boolean injectLinksAsTags, boolean injectBaggageAsTags) { + processTagsAndBaggageCount++; + super.processTagsAndBaggage(consumer, injectLinksAsTags, injectBaggageAsTags); + } + } + + private static class ByteArrayChannel implements WritableByteChannel { + private byte[] data = new byte[0]; + + @Override + public int write(ByteBuffer src) { + int len = src.remaining(); + byte[] incoming = new byte[len]; + src.get(incoming); + byte[] combined = new byte[data.length + incoming.length]; + System.arraycopy(data, 0, combined, 0, data.length); + System.arraycopy(incoming, 0, combined, data.length, incoming.length); + data = combined; + return len; + } + + byte[] bytes() { + return data; + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void close() {} + } + + private static class CapturedBody implements ByteBufferConsumer { + private final TraceMapperV1 mapper; + private byte[] payloadBytes; + + private CapturedBody(TraceMapperV1 mapper) { + this.mapper = mapper; + } + + @Override + public void accept(int messageCount, ByteBuffer buffer) { + Payload payload = mapper.newPayload().withBody(messageCount, buffer); + payloadBytes = serializePayload(payload); + mapper.reset(); + } + } + + /** Subclass to expose SpanLink's protected constructor for testing. */ + private static class TestSpanLink extends SpanLink { + TestSpanLink( + DDTraceId traceId, + long spanId, + byte traceFlags, + String traceState, + SpanAttributes attributes) { + super(traceId, spanId, traceFlags, traceState, attributes); + } + } + + // --- Helper methods --- + private static void verifyChunk( - MessageUnpacker unpacker, List expectedTrace, List stringTable) + MessageUnpacker unpacker, + List expectedTrace, + List stringTable) throws IOException { int chunkFieldCount = unpacker.unpackMapHeader(); assertEquals(6, chunkFieldCount); @@ -605,54 +1017,64 @@ private static void verifyChunk( Map chunkAttributes = null; byte[] traceId = null; Integer samplingMechanism = null; - boolean seenSpans = false; + List decodedSpans = null; for (int i = 0; i < chunkFieldCount; i++) { int fieldId = unpacker.unpackInt(); switch (fieldId) { - case ChunkField.PRIORITY: + case 1: priority = unpacker.unpackInt(); break; - case ChunkField.ORIGIN: + case 2: origin = readStreamingString(unpacker, stringTable); break; - case ChunkField.ATTRIBUTES: + case 3: chunkAttributes = readAttributes(unpacker, stringTable); break; - case ChunkField.SPANS: - verifySpans(unpacker, expectedTrace, stringTable); - seenSpans = true; + case 4: + decodedSpans = verifySpans(unpacker, expectedTrace, stringTable); break; - case ChunkField.TRACE_ID: - traceId = readBinary(unpacker); + case 6: + int traceIdLen = unpacker.unpackBinaryHeader(); + traceId = new byte[traceIdLen]; + unpacker.readPayload(traceId); break; - case ChunkField.SAMPLING_MECHANISM: + case 7: samplingMechanism = unpacker.unpackInt(); break; default: - fail("Unexpected chunk field id: " + fieldId); + Assertions.fail("Unexpected chunk field id: " + fieldId); } } assertNotNull(priority); assertNotNull(origin); assertNotNull(chunkAttributes); - assertTrue(seenSpans); + assertNotNull(decodedSpans); assertNotNull(traceId); assertNotNull(samplingMechanism); - PojoSpan firstSpan = expectedTrace.get(0); - assertEquals(firstSpan.samplingPriority(), priority.intValue()); + TraceGenerator.PojoSpan firstSpan = expectedTrace.get(0); + assertEquals(firstSpan.samplingPriority(), (int) priority); assertEqualsWithNullAsEmpty(firstSpan.getOrigin(), origin); assertEquals(1, chunkAttributes.size()); assertEqualsWithNullAsEmpty( firstSpan.getLocalRootSpan().getServiceName(), (String) chunkAttributes.get("service")); assertArrayEquals(traceIdBytes(firstSpan.getTraceId()), traceId); - assertEquals(expectedSamplingMechanism(firstSpan.getBaggage()), samplingMechanism.intValue()); + assertEquals(expectedSamplingMechanism(firstSpan.getTags()), (int) samplingMechanism); + } + + private static byte[] traceIdBytes(DDTraceId traceId) { + return ByteBuffer.allocate(16) + .putLong(traceId.toHighOrderLong()) + .putLong(traceId.toLong()) + .array(); } - private static void verifySpans( - MessageUnpacker unpacker, List expectedTrace, List stringTable) + private static List verifySpans( + MessageUnpacker unpacker, + List expectedTrace, + List stringTable) throws IOException { int spanCount = unpacker.unpackArrayHeader(); assertEquals(expectedTrace.size(), spanCount); @@ -660,23 +1082,23 @@ private static void verifySpans( for (int i = 0; i < spanCount; i++) { verifySpan(unpacker, expectedTrace.get(i), stringTable); } + return expectedTrace; } private static void verifySpan( - MessageUnpacker unpacker, PojoSpan expectedSpan, List stringTable) + MessageUnpacker unpacker, TraceGenerator.PojoSpan expectedSpan, List stringTable) throws IOException { int spanFieldCount = unpacker.unpackMapHeader(); - assertEquals(EXPECTED_SPAN_FIELD_IDS.size(), spanFieldCount); + assertEquals(16, spanFieldCount); - Set spanFieldsSeen = new HashSet<>(); String service = null; String name = null; String resource = null; - long spanId = 0; - long parentId = 0; - long start = 0; - long duration = 0; - boolean error = false; + Long spanId = null; + Long parentId = null; + Long start = null; + Long duration = null; + Boolean error = null; Map attributes = null; String type = null; int linksCount = -1; @@ -684,101 +1106,96 @@ private static void verifySpan( String env = null; String version = null; String component = null; - int spanKind = -1; + Integer spanKind = null; for (int i = 0; i < spanFieldCount; i++) { int fieldId = unpacker.unpackInt(); - spanFieldsSeen.add(fieldId); switch (fieldId) { - case SpanField.SERVICE: + case 1: service = readStreamingString(unpacker, stringTable); break; - case SpanField.NAME: + case 2: name = readStreamingString(unpacker, stringTable); break; - case SpanField.RESOURCE: + case 3: resource = readStreamingString(unpacker, stringTable); break; - case SpanField.SPAN_ID: + case 4: spanId = unpackUnsignedLong(unpacker); break; - case SpanField.PARENT_ID: + case 5: parentId = unpackUnsignedLong(unpacker); break; - case SpanField.START: + case 6: start = unpacker.unpackLong(); break; - case SpanField.DURATION: + case 7: duration = unpacker.unpackLong(); break; - case SpanField.ERROR: + case 8: error = unpacker.unpackBoolean(); break; - case SpanField.ATTRIBUTES: + case 9: attributes = readAttributes(unpacker, stringTable); break; - case SpanField.TYPE: + case 10: type = readStreamingString(unpacker, stringTable); break; - case SpanField.LINKS: + case 11: linksCount = unpacker.unpackArrayHeader(); break; - case SpanField.EVENTS: + case 12: eventsCount = unpacker.unpackArrayHeader(); break; - case SpanField.ENV: + case 13: env = readStreamingString(unpacker, stringTable); break; - case SpanField.VERSION: + case 14: version = readStreamingString(unpacker, stringTable); break; - case SpanField.COMPONENT: + case 15: component = readStreamingString(unpacker, stringTable); break; - case SpanField.KIND: + case 16: spanKind = unpacker.unpackInt(); break; default: - fail("Unexpected span field id: " + fieldId); + Assertions.fail("Unexpected span field id: " + fieldId); } } - // A 16-entry map could still repeat one field id and omit another, which would leave a decoded - // value at its initial sentinel. Pinning the id set makes each field present exactly once, so - // the value assertions below cannot pass on an unwritten field (e.g. parentId 0, error false). - assertEquals(EXPECTED_SPAN_FIELD_IDS, spanFieldsSeen); - assertEqualsWithNullAsEmpty(expectedSpan.getServiceName(), service); assertEqualsWithNullAsEmpty(expectedSpan.getOperationName(), name); assertEqualsWithNullAsEmpty(expectedSpan.getResourceName(), resource); - assertEquals(expectedSpan.getSpanId(), spanId); - assertEquals(expectedSpan.getParentId(), parentId); - assertEquals(expectedSpan.getStartTime(), start); - assertEquals(expectedSpan.getDurationNano(), duration); + assertEquals(expectedSpan.getSpanId(), (long) spanId); + assertEquals(expectedSpan.getParentId(), (long) parentId); + assertEquals(expectedSpan.getStartTime(), (long) start); + assertEquals(expectedSpan.getDurationNano(), (long) duration); assertEquals(expectedSpan.getError() != 0, error); assertEqualsWithNullAsEmpty(expectedSpan.getType(), type); assertEquals(0, linksCount); assertEquals(0, eventsCount); - assertEqualsWithNullAsEmpty(expectedSpan.getTag(ENV), env); - assertEqualsWithNullAsEmpty(expectedSpan.getTag(VERSION), version); - assertEqualsWithNullAsEmpty(expectedSpan.getTag(COMPONENT), component); - assertEquals(TraceMapperV1.getSpanKindValue(expectedSpan.getTag(SPAN_KIND)), spanKind); + assertEqualsWithNullAsEmpty(expectedSpan.getTag(Tags.ENV), env); + assertEqualsWithNullAsEmpty(expectedSpan.getTag(Tags.VERSION), version); + assertEqualsWithNullAsEmpty(expectedSpan.getTag(Tags.COMPONENT), component); + assertEquals( + TraceMapperV1.getSpanKindValue(expectedSpan.getTag(Tags.SPAN_KIND)), (int) spanKind); assertNotNull(attributes); int expectedHttpStatusCode = expectedSpan.getHttpStatusCode(); boolean shouldContainHttpStatus = - expectedHttpStatusCode != 0 && !expectedSpan.getTags().containsKey(HTTP_STATUS); + expectedHttpStatusCode != 0 && !expectedSpan.getTags().containsKey("http.status_code"); Map expectedAttributes = new HashMap<>(expectedSpan.getBaggage()); - expectedAttributes.put(THREAD_ID, expectedSpan.getTag(THREAD_ID)); - expectedAttributes.put(THREAD_NAME, expectedSpan.getTag(THREAD_NAME)); + expectedAttributes.put(DDTags.THREAD_ID, expectedSpan.getTag(DDTags.THREAD_ID)); + expectedAttributes.put(DDTags.THREAD_NAME, expectedSpan.getTag(DDTags.THREAD_NAME)); for (Map.Entry entry : expectedSpan.getTags().entrySet()) { - if (SPAN_EVENTS.equals(entry.getKey())) { + if (DDTags.SPAN_EVENTS.equals(entry.getKey())) { continue; } addFlattenedExpectedAttribute(expectedAttributes, entry.getKey(), entry.getValue()); } if (shouldContainHttpStatus) { - expectedAttributes.put(HTTP_STATUS, Integer.toString(expectedHttpStatusCode)); + expectedAttributes.put("http.status_code", Integer.toString(expectedHttpStatusCode)); } if (expectedSpan.isTopLevel()) { expectedAttributes.put(InstrumentationTags.DD_TOP_LEVEL.toString(), 1d); @@ -787,11 +1204,47 @@ private static void verifySpan( assertEquals(expectedAttributes.size(), attributes.size()); for (Map.Entry entry : expectedAttributes.entrySet()) { String key = entry.getKey(); + Object expectedValue = entry.getValue(); assertTrue(attributes.containsKey(key), "Missing attribute key: " + key); - assertAttributeValueEquals(entry.getValue(), attributes.get(key), key); + assertAttributeValueEquals(expectedValue, attributes.get(key), key); } } + private static Map readAttributes( + MessageUnpacker unpacker, List stringTable) throws IOException { + int attrArraySize = unpacker.unpackArrayHeader(); + assertEquals(0, attrArraySize % 3); + int attrCount = attrArraySize / 3; + + Map attributes = new HashMap<>(); + for (int i = 0; i < attrCount; i++) { + String key = readStreamingString(unpacker, stringTable); + int attrType = unpacker.unpackInt(); + Object value; + switch (attrType) { + case TraceMapperV1.VALUE_TYPE_STRING: + value = readStreamingString(unpacker, stringTable); + break; + case TraceMapperV1.VALUE_TYPE_BOOLEAN: + value = unpacker.unpackBoolean(); + break; + case TraceMapperV1.VALUE_TYPE_FLOAT: + value = unpacker.unpackDouble(); + break; + case TraceMapperV1.VALUE_TYPE_BYTES: + int len = unpacker.unpackBinaryHeader(); + byte[] data = new byte[len]; + unpacker.readPayload(data); + value = data; + break; + default: + value = Assertions.fail("Unknown attribute value type: " + attrType); + } + attributes.put(key, value); + } + return attributes; + } + private static void assertAttributeValueEquals(Object expected, Object actual, String key) { if (expected instanceof Number) { assertInstanceOf(Number.class, actual, "Attribute " + key + " should be numeric"); @@ -806,6 +1259,14 @@ private static void assertAttributeValueEquals(Object expected, Object actual, S } } + private static long unpackUnsignedLong(MessageUnpacker unpacker) throws IOException { + MessageFormat format = unpacker.getNextFormat(); + if (format == MessageFormat.UINT64) { + return DDSpanId.from(unpacker.unpackBigInteger().toString()); + } + return unpacker.unpackLong(); + } + private static void addFlattenedExpectedAttribute( Map expectedAttributes, String key, Object value) { if (!(value instanceof Map)) { @@ -818,162 +1279,549 @@ private static void addFlattenedExpectedAttribute( } } - private static int expectedSamplingMechanism(Map propagationMetadata) { - String decisionMaker = propagationMetadata.get(DECISION_MAKER_TAG); - if (decisionMaker == null) { + private static int expectedSamplingMechanism(Map tags) { + Object decisionMakerRaw = tags.get("_dd.p.dm"); + if (decisionMakerRaw == null) { return SamplingMechanism.DEFAULT; } + String decisionMaker = String.valueOf(decisionMakerRaw); try { - return Math.abs(Integer.parseInt(decisionMaker)); + int value = Integer.parseInt(decisionMaker); + return value < 0 ? -value : value; } catch (NumberFormatException ignored) { int separator = decisionMaker.lastIndexOf('-'); if (separator >= 0 && separator + 1 < decisionMaker.length()) { try { - return Math.abs(Integer.parseInt(decisionMaker.substring(separator + 1))); + int value = Integer.parseInt(decisionMaker.substring(separator + 1)); + return value < 0 ? -value : value; } catch (NumberFormatException ignoredAgain) { - // not a mechanism suffix either, fall through to the default + // fall through } } return SamplingMechanism.DEFAULT; } } - /** Asserts the next value is encoded as an unsigned 64-bit integer, and returns it. */ - private static long unpackUint64(MessageUnpacker unpacker) throws IOException { - assertEquals(MessageFormat.UINT64, unpacker.getNextFormat()); - return unpackUnsignedLong(unpacker); - } + private static String readStreamingString(MessageUnpacker unpacker, List stringTable) + throws IOException { + MessageFormat format = unpacker.getNextFormat(); + if (format == FIXSTR || format == STR8 || format == STR16 || format == STR32) { + String value = unpacker.unpackString(); + if (!stringTable.contains(value)) { + stringTable.add(value); + } + return value; + } - private static byte[] serializeV1Payload(PojoSpan span) { - return serializeV1Payload(singletonList(span)); + int index = unpacker.unpackInt(); + assertTrue(index >= 0 && index < stringTable.size(), "Invalid string-table index: " + index); + return stringTable.get(index); } - /** Maps a single trace and returns the complete v1 payload bytes. */ - private static byte[] serializeV1Payload(List trace) { - TraceMapperV1 mapper = new TraceMapperV1(); - CapturedBody capturedBody = new CapturedBody(mapper); - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(2 << 20, capturedBody)); + private static void skipPayloadField( + MessageUnpacker unpacker, int fieldId, List stringTable) throws IOException { + switch (fieldId) { + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + case 9: + readStreamingString(unpacker, stringTable); + break; + case 10: + readAttributes(unpacker, stringTable); + break; + default: + Assertions.fail("Unexpected payload field id while skipping: " + fieldId); + } + } - assertTrue(packer.format(trace, mapper)); - packer.flush(); + private static void skipChunkField( + MessageUnpacker unpacker, int fieldId, List stringTable) throws IOException { + switch (fieldId) { + case 1: + unpacker.unpackInt(); + break; + case 2: + readStreamingString(unpacker, stringTable); + break; + case 3: + readAttributes(unpacker, stringTable); + break; + case 4: + int spanCount = unpacker.unpackArrayHeader(); + for (int i = 0; i < spanCount; i++) { + skipSpan(unpacker, stringTable); + } + break; + case 5: + unpacker.unpackBoolean(); + break; + case 6: + int len = unpacker.unpackBinaryHeader(); + byte[] ignored = new byte[len]; + unpacker.readPayload(ignored); + break; + case 7: + unpacker.unpackInt(); + break; + default: + Assertions.fail("Unexpected chunk field id while skipping: " + fieldId); + } + } - assertNotNull(capturedBody.payloadBytes); - return capturedBody.payloadBytes; + private static void skipSpan(MessageUnpacker unpacker, List stringTable) + throws IOException { + int fieldCount = unpacker.unpackMapHeader(); + for (int i = 0; i < fieldCount; i++) { + int fieldId = unpacker.unpackInt(); + switch (fieldId) { + case 1: + case 2: + case 3: + case 10: + case 13: + case 14: + case 15: + readStreamingString(unpacker, stringTable); + break; + case 4: + case 5: + unpacker.unpackValue().asNumberValue().toLong(); + break; + case 6: + case 7: + unpacker.unpackLong(); + break; + case 8: + unpacker.unpackBoolean(); + break; + case 9: + int attrArraySize = unpacker.unpackArrayHeader(); + int attrCount = attrArraySize / 3; + for (int j = 0; j < attrCount; j++) { + readStreamingString(unpacker, stringTable); + int type = unpacker.unpackInt(); + switch (type) { + case TraceMapperV1.VALUE_TYPE_STRING: + readStreamingString(unpacker, stringTable); + break; + case TraceMapperV1.VALUE_TYPE_BOOLEAN: + unpacker.unpackBoolean(); + break; + case TraceMapperV1.VALUE_TYPE_FLOAT: + unpacker.unpackDouble(); + break; + case TraceMapperV1.VALUE_TYPE_BYTES: + int blen = unpacker.unpackBinaryHeader(); + byte[] bignored = new byte[blen]; + unpacker.readPayload(bignored); + break; + default: + Assertions.fail("Unexpected attribute type while skipping: " + type); + } + } + break; + case 11: + case 12: + unpacker.unpackArrayHeader(); + break; + case 16: + unpacker.unpackInt(); + break; + default: + Assertions.fail("Unexpected span field id while skipping: " + fieldId); + } + } } - private static PojoSpan span(Map tags) { - return span(123L, 0L, emptyMap(), tags, 200, emptyList()); + private static Map readFirstSpanAttributes( + MessageUnpacker unpacker, List stringTable) throws IOException { + int payloadFieldCount = unpacker.unpackMapHeader(); + for (int i = 0; i < payloadFieldCount; i++) { + int payloadFieldId = unpacker.unpackInt(); + if (payloadFieldId != 11) { + skipPayloadField(unpacker, payloadFieldId, stringTable); + continue; + } + + int chunkCount = unpacker.unpackArrayHeader(); + assertEquals(1, chunkCount); + + int chunkFieldCount = unpacker.unpackMapHeader(); + for (int chunkFieldIndex = 0; chunkFieldIndex < chunkFieldCount; chunkFieldIndex++) { + int chunkFieldId = unpacker.unpackInt(); + if (chunkFieldId != 4) { + skipChunkField(unpacker, chunkFieldId, stringTable); + continue; + } + + int spanCount = unpacker.unpackArrayHeader(); + assertEquals(1, spanCount); + + int spanFieldCount = unpacker.unpackMapHeader(); + for (int spanFieldIndex = 0; spanFieldIndex < spanFieldCount; spanFieldIndex++) { + int spanFieldId = unpacker.unpackInt(); + if (spanFieldId == 9) { + return readAttributes(unpacker, stringTable); + } + skipSpanField(unpacker, spanFieldId, stringTable); + } + } + } + return Assertions.fail("Could not find span attributes field in first span"); } - private static PojoSpan span(Map tags, int statusCode) { - return span(123L, 0L, emptyMap(), tags, statusCode, emptyList()); + private static List> readFirstSpanLinks( + MessageUnpacker unpacker, List stringTable) throws IOException { + int payloadFieldCount = unpacker.unpackMapHeader(); + for (int i = 0; i < payloadFieldCount; i++) { + int payloadFieldId = unpacker.unpackInt(); + if (payloadFieldId != 11) { + skipPayloadField(unpacker, payloadFieldId, stringTable); + continue; + } + + int chunkCount = unpacker.unpackArrayHeader(); + assertEquals(1, chunkCount); + + int chunkFieldCount = unpacker.unpackMapHeader(); + for (int chunkFieldIndex = 0; chunkFieldIndex < chunkFieldCount; chunkFieldIndex++) { + int chunkFieldId = unpacker.unpackInt(); + if (chunkFieldId != 4) { + skipChunkField(unpacker, chunkFieldId, stringTable); + continue; + } + + int spanCount = unpacker.unpackArrayHeader(); + assertEquals(1, spanCount); + + int spanFieldCount = unpacker.unpackMapHeader(); + for (int spanFieldIndex = 0; spanFieldIndex < spanFieldCount; spanFieldIndex++) { + int spanFieldId = unpacker.unpackInt(); + if (spanFieldId == 11) { + return readSpanLinks(unpacker, stringTable); + } + skipSpanField(unpacker, spanFieldId, stringTable); + } + } + } + return Assertions.fail("Could not find span links field in first span"); } - private static PojoSpan spanWithBaggage(Map baggage) { - return span(123L, 0L, baggage, emptyMap(), 200, emptyList()); + private static void skipSpanField(MessageUnpacker unpacker, int fieldId, List stringTable) + throws IOException { + switch (fieldId) { + case 1: + case 2: + case 3: + case 10: + case 13: + case 14: + case 15: + readStreamingString(unpacker, stringTable); + break; + case 4: + case 5: + unpacker.unpackValue().asNumberValue().toLong(); + break; + case 6: + case 7: + unpacker.unpackLong(); + break; + case 8: + unpacker.unpackBoolean(); + break; + case 9: + readAttributes(unpacker, stringTable); + break; + case 12: + int eventsCount = unpacker.unpackArrayHeader(); + for (int j = 0; j < eventsCount; j++) { + skipSpanEvent(unpacker, stringTable); + } + break; + case 11: + int linksCount = unpacker.unpackArrayHeader(); + for (int j = 0; j < linksCount; j++) { + int linkFieldCount = unpacker.unpackMapHeader(); + for (int k = 0; k < linkFieldCount; k++) { + int linkFieldId = unpacker.unpackInt(); + switch (linkFieldId) { + case 1: + int traceIdLen = unpacker.unpackBinaryHeader(); + byte[] traceIdIgnored = new byte[traceIdLen]; + unpacker.readPayload(traceIdIgnored); + break; + case 2: + case 5: + unpacker.unpackValue().asNumberValue().toLong(); + break; + case 3: + readAttributes(unpacker, stringTable); + break; + case 4: + readStreamingString(unpacker, stringTable); + break; + default: + Assertions.fail("Unexpected span link field id while skipping: " + linkFieldId); + } + } + } + break; + case 16: + unpacker.unpackInt(); + break; + default: + Assertions.fail("Unexpected span field id while skipping: " + fieldId); + } } - private static PojoSpan spanWithIds(long spanId, long parentId) { - return span(spanId, parentId, emptyMap(), emptyMap(), 200, emptyList()); + private static List> readSpanLinks( + MessageUnpacker unpacker, List stringTable) throws IOException { + int linksCount = unpacker.unpackArrayHeader(); + List> links = new ArrayList<>(); + + for (int i = 0; i < linksCount; i++) { + int linkFieldCount = unpacker.unpackMapHeader(); + assertEquals(5, linkFieldCount); + + byte[] traceId = null; + Long spanId = null; + Map attributes = null; + String tracestate = null; + Long flags = null; + + for (int j = 0; j < linkFieldCount; j++) { + int linkFieldId = unpacker.unpackInt(); + switch (linkFieldId) { + case 1: + int traceIdLen = unpacker.unpackBinaryHeader(); + traceId = new byte[traceIdLen]; + unpacker.readPayload(traceId); + break; + case 2: + spanId = unpacker.unpackValue().asNumberValue().toLong(); + break; + case 3: + attributes = readAttributes(unpacker, stringTable); + break; + case 4: + tracestate = readStreamingString(unpacker, stringTable); + break; + case 5: + flags = unpacker.unpackValue().asNumberValue().toLong(); + break; + default: + Assertions.fail("Unexpected span link field id: " + linkFieldId); + } + } + + Map link = new HashMap<>(); + link.put("traceId", traceId); + link.put("spanId", spanId); + link.put("attributes", attributes); + link.put("tracestate", tracestate); + link.put("flags", flags); + links.add(link); + } + + return links; } - private static PojoSpan spanWithLinks(List spanLinks) { - return span(123L, 0L, emptyMap(), emptyMap(), 200, spanLinks); + private static List> readFirstSpanEvents( + MessageUnpacker unpacker, List stringTable) throws IOException { + int payloadFieldCount = unpacker.unpackMapHeader(); + for (int i = 0; i < payloadFieldCount; i++) { + int payloadFieldId = unpacker.unpackInt(); + if (payloadFieldId != 11) { + skipPayloadField(unpacker, payloadFieldId, stringTable); + continue; + } + + int chunkCount = unpacker.unpackArrayHeader(); + assertEquals(1, chunkCount); + + int chunkFieldCount = unpacker.unpackMapHeader(); + for (int chunkFieldIndex = 0; chunkFieldIndex < chunkFieldCount; chunkFieldIndex++) { + int chunkFieldId = unpacker.unpackInt(); + if (chunkFieldId != 4) { + skipChunkField(unpacker, chunkFieldId, stringTable); + continue; + } + + int spanCount = unpacker.unpackArrayHeader(); + assertEquals(1, spanCount); + + int spanFieldCount = unpacker.unpackMapHeader(); + for (int spanFieldIndex = 0; spanFieldIndex < spanFieldCount; spanFieldIndex++) { + int spanFieldId = unpacker.unpackInt(); + if (spanFieldId == 12) { + return readSpanEvents(unpacker, stringTable); + } + skipSpanField(unpacker, spanFieldId, stringTable); + } + } + } + return Assertions.fail("Could not find span events field in first span"); } - /** A span carrying the fields shared by most tests; only the varying pieces are parameters. */ - private static PojoSpan span( - long spanId, - long parentId, - Map baggage, - Map tags, - int statusCode, - List spanLinks) { - return new PojoSpan( - "service-a", - "operation-a", - "resource-a", - DDTraceId.ONE, - spanId, - parentId, - 1000L, - 2000L, - 0, - baggage, - tags, - "web", - false, - SAMPLER_KEEP, - statusCode, - null, - spanLinks); + private static List> readSpanEvents( + MessageUnpacker unpacker, List stringTable) throws IOException { + int eventsCount = unpacker.unpackArrayHeader(); + List> events = new ArrayList<>(); + + for (int i = 0; i < eventsCount; i++) { + int eventFieldCount = unpacker.unpackMapHeader(); + assertEquals(3, eventFieldCount); + + Long timeUnixNano = null; + String name = null; + Map attributes = null; + + for (int j = 0; j < eventFieldCount; j++) { + int eventFieldId = unpacker.unpackInt(); + switch (eventFieldId) { + case 1: + timeUnixNano = unpacker.unpackLong(); + break; + case 2: + name = readStreamingString(unpacker, stringTable); + break; + case 3: + attributes = readEventAttributes(unpacker, stringTable); + break; + default: + Assertions.fail("Unexpected span event field id: " + eventFieldId); + } + } + + Map event = new HashMap<>(); + event.put("timeUnixNano", timeUnixNano); + event.put("name", name); + event.put("attributes", attributes); + events.add(event); + } + return events; } - private static final class CapturedBody implements ByteBufferConsumer { - private final TraceMapperV1 mapper; - private byte[] payloadBytes; + private static Map readEventAttributes( + MessageUnpacker unpacker, List stringTable) throws IOException { + int attrArraySize = unpacker.unpackArrayHeader(); + assertEquals(0, attrArraySize % 3); + int attrCount = attrArraySize / 3; + Map attributes = new HashMap<>(); + + for (int i = 0; i < attrCount; i++) { + String key = readStreamingString(unpacker, stringTable); + int attrType = unpacker.unpackInt(); + Object value; + switch (attrType) { + case TraceMapperV1.VALUE_TYPE_STRING: + value = readStreamingString(unpacker, stringTable); + break; + case TraceMapperV1.VALUE_TYPE_BOOLEAN: + value = unpacker.unpackBoolean(); + break; + case TraceMapperV1.VALUE_TYPE_FLOAT: + value = unpacker.unpackDouble(); + break; + case TraceMapperV1.VALUE_TYPE_INT: + value = unpacker.unpackLong(); + break; + case TraceMapperV1.VALUE_TYPE_ARRAY: + value = readEventArrayValue(unpacker, stringTable); + break; + default: + value = Assertions.fail("Unknown event attribute value type: " + attrType); + } + attributes.put(key, value); + } + return attributes; + } - private CapturedBody(TraceMapperV1 mapper) { - this.mapper = mapper; + private static List readEventArrayValue( + MessageUnpacker unpacker, List stringTable) throws IOException { + int itemArraySize = unpacker.unpackArrayHeader(); + assertEquals(0, itemArraySize % 2); + int itemCount = itemArraySize / 2; + List values = new ArrayList<>(); + for (int i = 0; i < itemCount; i++) { + int itemType = unpacker.unpackInt(); + switch (itemType) { + case TraceMapperV1.VALUE_TYPE_STRING: + values.add(readStreamingString(unpacker, stringTable)); + break; + case TraceMapperV1.VALUE_TYPE_BOOLEAN: + values.add(unpacker.unpackBoolean()); + break; + case TraceMapperV1.VALUE_TYPE_FLOAT: + values.add(unpacker.unpackDouble()); + break; + case TraceMapperV1.VALUE_TYPE_INT: + values.add(unpacker.unpackLong()); + break; + default: + Assertions.fail("Unknown event array item type: " + itemType); + } } + return values; + } - @Override - public void accept(int messageCount, ByteBuffer buffer) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - try { - Payload payload = mapper.newPayload().withBody(messageCount, buffer); - payload.writeTo(Channels.newChannel(out)); - payloadBytes = out.toByteArray(); - } catch (IOException e) { - fail("Failed to serialize the v1 payload: " + e.getMessage()); - } finally { - mapper.reset(); + private static void skipSpanEvent(MessageUnpacker unpacker, List stringTable) + throws IOException { + int fieldCount = unpacker.unpackMapHeader(); + for (int i = 0; i < fieldCount; i++) { + int fieldId = unpacker.unpackInt(); + switch (fieldId) { + case 1: + unpacker.unpackLong(); + break; + case 2: + readStreamingString(unpacker, stringTable); + break; + case 3: + readEventAttributes(unpacker, stringTable); + break; + default: + Assertions.fail("Unexpected event field id while skipping: " + fieldId); } } } - /** A span counting how many times its tags and baggage were handed to the mapper. */ - private static final class CountingPojoSpan extends PojoSpan { - private int processTagsAndBaggageCount = 0; + private static byte[] serializeMappedPayload( + TraceMapperV1 mapper, List> traces) { + CapturedBody capturedBody = new CapturedBody(mapper); + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(2 << 20, capturedBody)); - private CountingPojoSpan( - String operationName, CharSequence resourceName, long spanId, long parentId) { - super( - "service-a", - operationName, - resourceName, - DDTraceId.ONE, - spanId, - parentId, - 1000L, - 2000L, - 0, - emptyMap(), - singletonMap(HTTP_URL, "http://localhost:7777/"), - "web", - false, - SAMPLER_KEEP, - 200, - null); + for (List trace : traces) { + assertTrue(packer.format(trace, mapper)); } + packer.flush(); - @Override - public void processTagsAndBaggage(MetadataConsumer consumer) { - processTagsAndBaggageCount++; - super.processTagsAndBaggage(consumer); + assertNotNull(capturedBody.payloadBytes); + return capturedBody.payloadBytes; + } + + private static byte[] serializePayload(Payload payload) { + ByteArrayChannel channel = new ByteArrayChannel(); + try { + payload.writeTo(channel); + } catch (IOException e) { + Assertions.fail(e.getMessage()); } + return channel.bytes(); } - /** {@link SpanLink}'s constructor is protected, so tests reach it through a subclass. */ - private static final class TestSpanLink extends SpanLink { - private TestSpanLink( - DDTraceId traceId, - long spanId, - byte traceFlags, - String traceState, - SpanAttributes attributes) { - super(traceId, spanId, traceFlags, traceState, attributes); + private static void assertEqualsWithNullAsEmpty(CharSequence expected, String actual) { + if (expected == null) { + assertEquals("", actual); + } else { + assertEquals(expected.toString(), actual); } } } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.java new file mode 100644 index 00000000000..c3464d47d36 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.java @@ -0,0 +1,457 @@ +package datadog.trace.common.writer.ddintake; + +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT; +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import datadog.communication.serialization.ByteBufferConsumer; +import datadog.communication.serialization.FlushingBuffer; +import datadog.communication.serialization.msgpack.MsgPackWriter; +import datadog.trace.agent.test.server.http.JavaTestHttpServer; +import datadog.trace.api.DDTags; +import datadog.trace.api.civisibility.CiVisibilityWellKnownTags; +import datadog.trace.api.intake.TrackType; +import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; +import datadog.trace.bootstrap.instrumentation.api.ServiceNameSources; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.common.writer.Payload; +import datadog.trace.common.writer.RemoteApi; +import datadog.trace.common.writer.RemoteMapper; +import datadog.trace.core.DDCoreJavaSpecification; +import datadog.trace.core.DDSpan; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.zip.GZIPInputStream; +import okhttp3.HttpUrl; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.msgpack.jackson.dataformat.MessagePackFactory; + +@Timeout(20) +class DDEvpProxyApiTest extends DDCoreJavaSpecification { + + static final CiVisibilityWellKnownTags WELL_KNOWN_TAGS = + new CiVisibilityWellKnownTags( + "my-runtime-id", + "my-env", + "my-language", + "my-runtime-name", + "my-runtime-version", + "my-runtime-vendor", + "my-os-arch", + "my-os-platform", + "my-os-version", + "false"); + + static final String INTAKE_SUBDOMAIN = "citestcycle-intake"; + static final ObjectMapper MSG_PACK_MAPPER = new ObjectMapper(new MessagePackFactory()); + static final String DD_EVP_SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain"; + + @Test + void testSendingAnEmptyListOfTracesReturnsNoErrors() { + TrackType trackType = TrackType.CITESTCYCLE; + String apiVersion = "v2"; + String evpProxyEndpoint = V2_EVP_PROXY_ENDPOINT; + String path = buildAgentEvpProxyPath(evpProxyEndpoint, trackType, apiVersion); + JavaTestHttpServer agentEvpProxy = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.post( + path, + api -> { + if (!"application/msgpack" + .equals(api.getRequest().getContentType())) { + api.getResponse() + .status(400) + .send("wrong type: " + api.getRequest().getContentType()); + } else { + api.getResponse().status(200).send(); + } + }))); + DDEvpProxyApi client = + createEvpProxyApi( + agentEvpProxy.getAddress().toString(), evpProxyEndpoint, trackType, false); + Payload payload = prepareTraces(trackType, false, Collections.emptyList()); + + try { + RemoteApi.Response clientResponse = client.sendSerializedTraces(payload); + assertTrue(clientResponse.success()); + assertTrue(clientResponse.status().isPresent()); + assertEquals(200, clientResponse.status().getAsInt()); + assertEquals(path, agentEvpProxy.getLastRequest().getPath()); + assertEquals( + INTAKE_SUBDOMAIN, agentEvpProxy.getLastRequest().getHeader(DD_EVP_SUBDOMAIN_HEADER)); + } finally { + agentEvpProxy.close(); + } + } + + @Test + void testRetriesWhenBackendReturns5xx() { + TrackType trackType = TrackType.CITESTCYCLE; + String apiVersion = "v2"; + String evpProxyEndpoint = V2_EVP_PROXY_ENDPOINT; + String path = buildAgentEvpProxyPath(evpProxyEndpoint, trackType, apiVersion); + int[] retry = {1}; + JavaTestHttpServer agentEvpProxy = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.post( + path, + api -> { + if (retry[0] < 5) { + api.getResponse().status(503).send(); + retry[0]++; + } else { + api.getResponse().status(200).send(); + } + }))); + DDEvpProxyApi client = + createEvpProxyApi( + agentEvpProxy.getAddress().toString(), evpProxyEndpoint, trackType, false); + Payload payload = prepareTraces(trackType, false, Collections.emptyList()); + + try { + RemoteApi.Response clientResponse = client.sendSerializedTraces(payload); + assertTrue(clientResponse.success()); + assertTrue(clientResponse.status().isPresent()); + assertEquals(200, clientResponse.status().getAsInt()); + assertEquals(path, agentEvpProxy.getLastRequest().getPath()); + assertEquals( + INTAKE_SUBDOMAIN, agentEvpProxy.getLastRequest().getHeader(DD_EVP_SUBDOMAIN_HEADER)); + } finally { + agentEvpProxy.close(); + } + } + + @Test + void testContentIsSentAsMsgpackEmptyTraces() throws IOException { + runContentIsSentAsMsgpackTest( + TrackType.CITESTCYCLE, "v2", V2_EVP_PROXY_ENDPOINT, false, emptyList(), emptyMap()); + } + + @Test + void testContentIsSentAsMsgpackFakeTypeSpan() throws IOException { + DDSpan span = buildSpan(1L, "fakeType", Collections.singletonMap("service.name", "my-service")); + span.finish(); + setDurationNano(span, 10L); + List> traces = singletonList(singletonList(span)); + + Map metadata = buildMetadataMap(); + Map content = new TreeMap<>(); + content.put("service", "my-service"); + content.put("name", "fakeOperation"); + content.put("resource", "fakeResource"); + content.put("error", 0); + content.put("trace_id", 1L); + content.put("span_id", 1L); + content.put("parent_id", 0L); + content.put("start", 1000L); + content.put("duration", 10L); + content.put("meta", singletonMap(DDTags.DD_SVC_SRC, ServiceNameSources.MANUAL.toString())); + content.put("metrics", emptyMap()); + Map event = new TreeMap<>(); + event.put("type", "span"); + event.put("version", 1); + event.put("content", content); + Map expectedRequestBody = new TreeMap<>(); + expectedRequestBody.put("version", 1); + expectedRequestBody.put("metadata", metadata); + expectedRequestBody.put("events", singletonList(event)); + + runContentIsSentAsMsgpackTest( + TrackType.CITESTCYCLE, "v2", V2_EVP_PROXY_ENDPOINT, false, traces, expectedRequestBody); + } + + @Test + void testContentIsSentAsMsgpackTestSpan() throws IOException { + Map spanTags = new HashMap<>(); + spanTags.put("test_suite_id", 123L); + spanTags.put("test_module_id", 456L); + DDSpan span = buildSpan(1L, InternalSpanTypes.TEST, spanTags); + span.finish(); + setDurationNano(span, 10L); + List> traces = singletonList(singletonList(span)); + + Map metadata = buildMetadataMap(); + Map content = new TreeMap<>(); + content.put("test_suite_id", 123L); + content.put("test_module_id", 456L); + content.put("service", "fakeService"); + content.put("name", "fakeOperation"); + content.put("resource", "fakeResource"); + content.put("error", 0); + content.put("trace_id", 1L); + content.put("span_id", 1L); + content.put("parent_id", 0L); + content.put("start", 1000L); + content.put("duration", 10L); + content.put("meta", emptyMap()); + content.put("metrics", emptyMap()); + Map event = new TreeMap<>(); + event.put("type", "test"); + event.put("version", 2); + event.put("content", content); + Map expectedRequestBody = new TreeMap<>(); + expectedRequestBody.put("version", 1); + expectedRequestBody.put("metadata", metadata); + expectedRequestBody.put("events", singletonList(event)); + + runContentIsSentAsMsgpackTest( + TrackType.CITESTCYCLE, "v2", V2_EVP_PROXY_ENDPOINT, false, traces, expectedRequestBody); + } + + @Test + void testContentIsSentAsMsgpackTestSuiteEndSpan() throws IOException { + Map spanTags = new HashMap<>(); + spanTags.put("test_suite_id", 123L); + spanTags.put("test_module_id", 456L); + DDSpan span = buildSpan(1L, InternalSpanTypes.TEST_SUITE_END, spanTags); + span.finish(); + setDurationNano(span, 10L); + List> traces = singletonList(singletonList(span)); + + Map metadata = buildMetadataMap(); + Map content = new TreeMap<>(); + content.put("test_suite_id", 123L); + content.put("test_module_id", 456L); + content.put("service", "fakeService"); + content.put("name", "fakeOperation"); + content.put("resource", "fakeResource"); + content.put("error", 0); + content.put("start", 1000L); + content.put("duration", 10L); + content.put("meta", emptyMap()); + content.put("metrics", emptyMap()); + Map event = new TreeMap<>(); + event.put("type", "test_suite_end"); + event.put("version", 1); + event.put("content", content); + Map expectedRequestBody = new TreeMap<>(); + expectedRequestBody.put("version", 1); + expectedRequestBody.put("metadata", metadata); + expectedRequestBody.put("events", singletonList(event)); + + runContentIsSentAsMsgpackTest( + TrackType.CITESTCYCLE, "v2", V2_EVP_PROXY_ENDPOINT, false, traces, expectedRequestBody); + } + + @Test + void testContentIsSentAsMsgpackTestModuleEndSpanWithCompression() throws IOException { + Map spanTags = new HashMap<>(); + spanTags.put("test_module_id", 456L); + DDSpan span = buildSpan(1L, InternalSpanTypes.TEST_MODULE_END, spanTags); + span.finish(); + setDurationNano(span, 10L); + List> traces = singletonList(singletonList(span)); + + Map metadata = buildMetadataMap(); + Map content = new TreeMap<>(); + content.put("test_module_id", 456L); + content.put("service", "fakeService"); + content.put("name", "fakeOperation"); + content.put("resource", "fakeResource"); + content.put("error", 0); + content.put("start", 1000L); + content.put("duration", 10L); + content.put("meta", emptyMap()); + content.put("metrics", emptyMap()); + Map event = new TreeMap<>(); + event.put("type", "test_module_end"); + event.put("version", 1); + event.put("content", content); + Map expectedRequestBody = new TreeMap<>(); + expectedRequestBody.put("version", 1); + expectedRequestBody.put("metadata", metadata); + expectedRequestBody.put("events", singletonList(event)); + + runContentIsSentAsMsgpackTest( + TrackType.CITESTCYCLE, "v2", V4_EVP_PROXY_ENDPOINT, true, traces, expectedRequestBody); + } + + // --- Helper methods --- + + private void runContentIsSentAsMsgpackTest( + TrackType trackType, + String apiVersion, + String evpProxyEndpoint, + boolean compressionEnabled, + List> traces, + Map expectedRequestBody) + throws IOException { + String path = buildAgentEvpProxyPath(evpProxyEndpoint, trackType, apiVersion); + JavaTestHttpServer agentEvpProxy = + JavaTestHttpServer.httpServer( + s -> s.handlers(h -> h.post(path, api -> api.getResponse().send()))); + DDEvpProxyApi client = + createEvpProxyApi( + agentEvpProxy.getAddress().toString(), evpProxyEndpoint, trackType, compressionEnabled); + Payload payload = prepareTraces(trackType, compressionEnabled, traces); + + try { + client.sendSerializedTraces(payload).status(); + assertEquals("application/msgpack", agentEvpProxy.getLastRequest().getContentType()); + Map actualBody = + convertMap(agentEvpProxy.getLastRequest().getBody(), compressionEnabled); + assertDeepEquals(expectedRequestBody, actualBody); + } finally { + agentEvpProxy.close(); + } + } + + private static Map buildMetadataMap() { + Map star = new TreeMap<>(); + star.put("env", "my-env"); + star.put("runtime-id", "my-runtime-id"); + star.put("language", "my-language"); + star.put(Tags.RUNTIME_NAME, "my-runtime-name"); + star.put(Tags.RUNTIME_VERSION, "my-runtime-version"); + star.put(Tags.RUNTIME_VENDOR, "my-runtime-vendor"); + star.put(Tags.OS_ARCHITECTURE, "my-os-arch"); + star.put(Tags.OS_PLATFORM, "my-os-platform"); + star.put(Tags.OS_VERSION, "my-os-version"); + star.put(DDTags.TEST_IS_USER_PROVIDED_SERVICE, "false"); + Map metadata = new TreeMap<>(); + metadata.put("*", star); + return metadata; + } + + private static void setDurationNano(DDSpan span, long duration) { + try { + Field field = DDSpan.class.getDeclaredField("durationNano"); + field.setAccessible(true); + field.setLong(span, duration); + } catch (NoSuchFieldException | IllegalAccessException e) { + Assertions.fail("Could not set durationNano: " + e.getMessage()); + } + } + + static Map convertMap(byte[] bytes, boolean compressionEnabled) + throws IOException { + if (compressionEnabled) { + bytes = decompress(bytes); + } + return MSG_PACK_MAPPER.readValue(bytes, new TypeReference>() {}); + } + + static byte[] decompress(byte[] bytes) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (GZIPInputStream zip = new GZIPInputStream(new ByteArrayInputStream(bytes))) { + byte[] buf = new byte[4096]; + int len; + while ((len = zip.read(buf)) != -1) { + baos.write(buf, 0, len); + } + } + return baos.toByteArray(); + } + + private DDEvpProxyApi createEvpProxyApi( + String agentUrl, String evpProxyEndpoint, TrackType trackType, boolean compressionEnabled) { + return DDEvpProxyApi.builder() + .agentUrl(HttpUrl.get(agentUrl)) + .evpProxyEndpoint(evpProxyEndpoint) + .trackType(trackType) + .compressionEnabled(compressionEnabled) + .build(); + } + + private RemoteMapper discoverMapper(TrackType trackType, boolean compressionEnabled) { + DDIntakeMapperDiscovery mapperDiscovery = + new DDIntakeMapperDiscovery(trackType, WELL_KNOWN_TAGS, compressionEnabled); + mapperDiscovery.discover(); + return mapperDiscovery.getMapper(); + } + + private String buildAgentEvpProxyPath( + String evpProxyEndpoint, TrackType trackType, String apiVersion) { + return "/" + evpProxyEndpoint + "api/" + apiVersion + "/" + trackType.name().toLowerCase(); + } + + private Payload prepareTraces( + TrackType trackType, boolean compressionEnabled, List> traces) { + TracesCapture traceCapture = new TracesCapture(); + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1 << 20, traceCapture)); + RemoteMapper mapper = discoverMapper(trackType, compressionEnabled); + for (List trace : traces) { + packer.format(trace, mapper); + } + packer.flush(); + return mapper + .newPayload() + .withBody( + traceCapture.traceCount, + traces.isEmpty() ? ByteBuffer.allocate(0) : traceCapture.buffer); + } + + @SuppressWarnings("unchecked") + static void assertDeepEquals(Object expected, Object actual) { + if (expected == null && actual == null) { + return; + } + if (expected == null || actual == null) { + Assertions.fail("Expected " + expected + " but got " + actual); + } + if (expected instanceof Map) { + assertInstanceOf(Map.class, actual, "Expected Map but got " + actual.getClass()); + Map expectedMap = (Map) expected; + Map actualMap = (Map) actual; + assertEquals(expectedMap.size(), actualMap.size(), "Map size mismatch"); + for (Map.Entry entry : expectedMap.entrySet()) { + assertTrue(actualMap.containsKey(entry.getKey()), "Missing key: " + entry.getKey()); + assertDeepEquals(entry.getValue(), actualMap.get(entry.getKey())); + } + } else if (expected instanceof List) { + assertInstanceOf(List.class, actual, "Expected List but got " + actual.getClass()); + List expectedList = (List) expected; + List actualList = (List) actual; + assertEquals(expectedList.size(), actualList.size(), "List size mismatch"); + for (int i = 0; i < expectedList.size(); i++) { + assertDeepEquals(expectedList.get(i), actualList.get(i)); + } + } else if (expected instanceof Number && actual instanceof Number) { + if (expected instanceof Float + || expected instanceof Double + || actual instanceof Float + || actual instanceof Double) { + assertEquals(((Number) expected).doubleValue(), ((Number) actual).doubleValue(), 0.0001); + } else { + assertEquals(((Number) expected).longValue(), ((Number) actual).longValue()); + } + } else { + assertEquals(expected, actual); + } + } + + static class TracesCapture implements ByteBufferConsumer { + int traceCount; + ByteBuffer buffer; + + @Override + public void accept(int messageCount, ByteBuffer buffer) { + this.buffer = buffer; + this.traceCount = messageCount; + } + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeApiTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeApiTest.java new file mode 100644 index 00000000000..4609c585a65 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeApiTest.java @@ -0,0 +1,463 @@ +package datadog.trace.common.writer.ddintake; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import datadog.communication.serialization.ByteBufferConsumer; +import datadog.communication.serialization.FlushingBuffer; +import datadog.communication.serialization.msgpack.MsgPackWriter; +import datadog.trace.agent.test.server.http.JavaTestHttpServer; +import datadog.trace.api.DDTags; +import datadog.trace.api.civisibility.CiVisibilityWellKnownTags; +import datadog.trace.api.intake.TrackType; +import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; +import datadog.trace.bootstrap.instrumentation.api.ServiceNameSources; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.common.writer.Payload; +import datadog.trace.common.writer.RemoteApi; +import datadog.trace.common.writer.RemoteMapper; +import datadog.trace.core.DDCoreJavaSpecification; +import datadog.trace.core.DDSpan; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.zip.GZIPInputStream; +import okhttp3.HttpUrl; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.msgpack.jackson.dataformat.MessagePackFactory; + +@Timeout(20) +class DDIntakeApiTest extends DDCoreJavaSpecification { + + static final CiVisibilityWellKnownTags WELL_KNOWN_TAGS = + new CiVisibilityWellKnownTags( + "my-runtime-id", + "my-env", + "my-language", + "my-runtime-name", + "my-runtime-version", + "my-runtime-vendor", + "my-os-arch", + "my-os-platform", + "my-os-version", + "false"); + + static final String API_KEY = "my-secret-apikey"; + static final ObjectMapper MSG_PACK_MAPPER = new ObjectMapper(new MessagePackFactory()); + + @Test + void testSendingAnEmptyListOfTracesReturnsNoErrors() { + TrackType trackType = TrackType.CITESTCYCLE; + String apiVersion = "v2"; + String path = buildIntakePath(trackType, apiVersion); + JavaTestHttpServer intake = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.post( + path, + api -> { + if (!"application/msgpack" + .equals(api.getRequest().getContentType())) { + api.getResponse() + .status(400) + .send("wrong type: " + api.getRequest().getContentType()); + } else { + api.getResponse().status(200).send(); + } + }))); + DDIntakeApi client = createIntakeApi(intake.getAddress().toString(), trackType); + Payload payload = prepareTraces(trackType, Collections.emptyList()); + + try { + RemoteApi.Response clientResponse = client.sendSerializedTraces(payload); + assertTrue(clientResponse.success()); + assertTrue(clientResponse.status().isPresent()); + assertEquals(200, clientResponse.status().getAsInt()); + assertEquals(path, intake.getLastRequest().getPath()); + } finally { + intake.close(); + } + } + + @Test + void testRetriesWhenBackendReturns5xx() { + TrackType trackType = TrackType.CITESTCYCLE; + String apiVersion = "v2"; + String path = buildIntakePath(trackType, apiVersion); + int[] retry = {1}; + JavaTestHttpServer intake = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.post( + path, + api -> { + if (retry[0] < 5) { + api.getResponse().status(503).send(); + retry[0]++; + } else { + api.getResponse().status(200).send(); + } + }))); + DDIntakeApi client = createIntakeApi(intake.getAddress().toString(), trackType); + Payload payload = prepareTraces(trackType, Collections.emptyList()); + + try { + RemoteApi.Response clientResponse = client.sendSerializedTraces(payload); + assertTrue(clientResponse.success()); + assertTrue(clientResponse.status().isPresent()); + assertEquals(200, clientResponse.status().getAsInt()); + assertEquals(path, intake.getLastRequest().getPath()); + } finally { + intake.close(); + } + } + + @Test + void testRetriesWhenBackendReturns429TooManyRequests() { + TrackType trackType = TrackType.CITESTCYCLE; + String apiVersion = "v2"; + String path = buildIntakePath(trackType, apiVersion); + int[] retry = {0}; + JavaTestHttpServer intake = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.post( + path, + api -> { + if (retry[0] < 1) { + api.getResponse() + .status(429) + .addHeader("x-ratelimit-reset", "0") + .send(); + retry[0]++; + } else { + api.getResponse().status(200).send(); + } + }))); + DDIntakeApi client = createIntakeApi(intake.getAddress().toString(), trackType); + Payload payload = prepareTraces(trackType, Collections.emptyList()); + + try { + RemoteApi.Response clientResponse = client.sendSerializedTraces(payload); + assertTrue(clientResponse.success()); + assertTrue(clientResponse.status().isPresent()); + assertEquals(200, clientResponse.status().getAsInt()); + assertEquals(path, intake.getLastRequest().getPath()); + } finally { + intake.close(); + } + } + + @Test + void testContentIsSentAsMsgpackEmptyTraces() throws IOException { + runContentIsSentAsMsgpackTest(TrackType.CITESTCYCLE, "v2", emptyList(), emptyMap()); + } + + @Test + void testContentIsSentAsMsgpackFakeTypeSpan() throws IOException { + DDSpan span = buildSpan(1L, "fakeType", Collections.singletonMap("service.name", "my-service")); + span.finish(); + setDurationNano(span, 10L); + List> traces = Collections.singletonList(Collections.singletonList(span)); + + Map metadata = buildMetadataMap(); + Map content = new TreeMap<>(); + content.put("service", "my-service"); + content.put("name", "fakeOperation"); + content.put("resource", "fakeResource"); + content.put("error", 0); + content.put("trace_id", 1L); + content.put("span_id", 1L); + content.put("parent_id", 0L); + content.put("start", 1000L); + content.put("duration", 10L); + content.put( + "meta", Collections.singletonMap(DDTags.DD_SVC_SRC, ServiceNameSources.MANUAL.toString())); + content.put("metrics", emptyMap()); + Map event = new TreeMap<>(); + event.put("type", "span"); + event.put("version", 1); + event.put("content", content); + Map expectedRequestBody = new TreeMap<>(); + expectedRequestBody.put("version", 1); + expectedRequestBody.put("metadata", metadata); + expectedRequestBody.put("events", Collections.singletonList(event)); + + runContentIsSentAsMsgpackTest(TrackType.CITESTCYCLE, "v2", traces, expectedRequestBody); + } + + @Test + void testContentIsSentAsMsgpackTestSpan() throws IOException { + Map spanTags = new HashMap<>(); + spanTags.put("test_suite_id", 123L); + spanTags.put("test_module_id", 456L); + DDSpan span = buildSpan(1L, InternalSpanTypes.TEST, spanTags); + span.finish(); + setDurationNano(span, 10L); + List> traces = Collections.singletonList(Collections.singletonList(span)); + + Map metadata = buildMetadataMap(); + Map content = new TreeMap<>(); + content.put("test_suite_id", 123L); + content.put("test_module_id", 456L); + content.put("service", "fakeService"); + content.put("name", "fakeOperation"); + content.put("resource", "fakeResource"); + content.put("error", 0); + content.put("trace_id", 1L); + content.put("span_id", 1L); + content.put("parent_id", 0L); + content.put("start", 1000L); + content.put("duration", 10L); + content.put("meta", emptyMap()); + content.put("metrics", emptyMap()); + Map event = new TreeMap<>(); + event.put("type", "test"); + event.put("version", 2); + event.put("content", content); + Map expectedRequestBody = new TreeMap<>(); + expectedRequestBody.put("version", 1); + expectedRequestBody.put("metadata", metadata); + expectedRequestBody.put("events", Collections.singletonList(event)); + + runContentIsSentAsMsgpackTest(TrackType.CITESTCYCLE, "v2", traces, expectedRequestBody); + } + + @Test + void testContentIsSentAsMsgpackTestSuiteEndSpan() throws IOException { + Map spanTags = new HashMap<>(); + spanTags.put("test_suite_id", 123L); + spanTags.put("test_module_id", 456L); + DDSpan span = buildSpan(1L, InternalSpanTypes.TEST_SUITE_END, spanTags); + span.finish(); + setDurationNano(span, 10L); + List> traces = Collections.singletonList(Collections.singletonList(span)); + + Map metadata = buildMetadataMap(); + Map content = new TreeMap<>(); + content.put("test_suite_id", 123L); + content.put("test_module_id", 456L); + content.put("service", "fakeService"); + content.put("name", "fakeOperation"); + content.put("resource", "fakeResource"); + content.put("error", 0); + content.put("start", 1000L); + content.put("duration", 10L); + content.put("meta", emptyMap()); + content.put("metrics", emptyMap()); + Map event = new TreeMap<>(); + event.put("type", "test_suite_end"); + event.put("version", 1); + event.put("content", content); + Map expectedRequestBody = new TreeMap<>(); + expectedRequestBody.put("version", 1); + expectedRequestBody.put("metadata", metadata); + expectedRequestBody.put("events", Collections.singletonList(event)); + + runContentIsSentAsMsgpackTest(TrackType.CITESTCYCLE, "v2", traces, expectedRequestBody); + } + + @Test + void testContentIsSentAsMsgpackTestModuleEndSpan() throws IOException { + DDSpan span = + buildSpan( + 1L, + InternalSpanTypes.TEST_MODULE_END, + Collections.singletonMap("test_module_id", 456L)); + span.finish(); + setDurationNano(span, 10L); + List> traces = Collections.singletonList(Collections.singletonList(span)); + + Map metadata = buildMetadataMap(); + Map content = new TreeMap<>(); + content.put("test_module_id", 456L); + content.put("service", "fakeService"); + content.put("name", "fakeOperation"); + content.put("resource", "fakeResource"); + content.put("error", 0); + content.put("start", 1000L); + content.put("duration", 10L); + content.put("meta", emptyMap()); + content.put("metrics", emptyMap()); + Map event = new TreeMap<>(); + event.put("type", "test_module_end"); + event.put("version", 1); + event.put("content", content); + Map expectedRequestBody = new TreeMap<>(); + expectedRequestBody.put("version", 1); + expectedRequestBody.put("metadata", metadata); + expectedRequestBody.put("events", Collections.singletonList(event)); + + runContentIsSentAsMsgpackTest(TrackType.CITESTCYCLE, "v2", traces, expectedRequestBody); + } + + // --- Helper methods --- + + private void runContentIsSentAsMsgpackTest( + TrackType trackType, + String apiVersion, + List> traces, + Map expectedRequestBody) + throws IOException { + String path = buildIntakePath(trackType, apiVersion); + JavaTestHttpServer intake = + JavaTestHttpServer.httpServer( + s -> s.handlers(h -> h.post(path, api -> api.getResponse().send()))); + DDIntakeApi client = createIntakeApi(intake.getAddress().toString(), trackType); + Payload payload = prepareTraces(trackType, traces); + + try { + client.sendSerializedTraces(payload).status(); + assertEquals("application/msgpack", intake.getLastRequest().getContentType()); + Map actualBody = convertMap(intake.getLastRequest().getBody()); + assertDeepEquals(expectedRequestBody, actualBody); + } finally { + intake.close(); + } + } + + private static Map buildMetadataMap() { + Map star = new TreeMap<>(); + star.put("env", "my-env"); + star.put("runtime-id", "my-runtime-id"); + star.put("language", "my-language"); + star.put(Tags.RUNTIME_NAME, "my-runtime-name"); + star.put(Tags.RUNTIME_VERSION, "my-runtime-version"); + star.put(Tags.RUNTIME_VENDOR, "my-runtime-vendor"); + star.put(Tags.OS_ARCHITECTURE, "my-os-arch"); + star.put(Tags.OS_PLATFORM, "my-os-platform"); + star.put(Tags.OS_VERSION, "my-os-version"); + star.put(DDTags.TEST_IS_USER_PROVIDED_SERVICE, "false"); + Map metadata = new TreeMap<>(); + metadata.put("*", star); + return metadata; + } + + private static void setDurationNano(DDSpan span, long duration) { + try { + Field field = DDSpan.class.getDeclaredField("durationNano"); + field.setAccessible(true); + field.setLong(span, duration); + } catch (NoSuchFieldException | IllegalAccessException e) { + Assertions.fail("Could not set durationNano: " + e.getMessage()); + } + } + + static Map convertMap(byte[] bytes) throws IOException { + return MSG_PACK_MAPPER.readValue( + decompress(bytes), new TypeReference>() {}); + } + + static byte[] decompress(byte[] bytes) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (GZIPInputStream zip = new GZIPInputStream(new ByteArrayInputStream(bytes))) { + byte[] buf = new byte[4096]; + int len; + while ((len = zip.read(buf)) != -1) { + baos.write(buf, 0, len); + } + } + return baos.toByteArray(); + } + + private DDIntakeApi createIntakeApi(String url, TrackType trackType) { + HttpUrl hostUrl = HttpUrl.get(url); + return DDIntakeApi.builder().hostUrl(hostUrl).trackType(trackType).apiKey(API_KEY).build(); + } + + private RemoteMapper discoverMapper(TrackType trackType) { + DDIntakeMapperDiscovery mapperDiscovery = + new DDIntakeMapperDiscovery(trackType, WELL_KNOWN_TAGS, true); + mapperDiscovery.discover(); + return mapperDiscovery.getMapper(); + } + + private String buildIntakePath(TrackType trackType, String apiVersion) { + return String.format("/api/%s/%s", apiVersion, trackType.name().toLowerCase()); + } + + private Payload prepareTraces(TrackType trackType, List> traces) { + TracesCapture traceCapture = new TracesCapture(); + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1 << 20, traceCapture)); + RemoteMapper mapper = discoverMapper(trackType); + for (List trace : traces) { + packer.format(trace, mapper); + } + packer.flush(); + return mapper + .newPayload() + .withBody( + traceCapture.traceCount, + traces.isEmpty() ? ByteBuffer.allocate(0) : traceCapture.buffer); + } + + @SuppressWarnings("unchecked") + static void assertDeepEquals(Object expected, Object actual) { + if (expected == null && actual == null) { + return; + } + if (expected == null || actual == null) { + Assertions.fail("Expected " + expected + " but got " + actual); + } + if (expected instanceof Map) { + assertInstanceOf(Map.class, actual, "Expected Map but got " + actual.getClass()); + Map expectedMap = (Map) expected; + Map actualMap = (Map) actual; + assertEquals(expectedMap.size(), actualMap.size(), "Map size mismatch"); + for (Map.Entry entry : expectedMap.entrySet()) { + assertTrue(actualMap.containsKey(entry.getKey()), "Missing key: " + entry.getKey()); + assertDeepEquals(entry.getValue(), actualMap.get(entry.getKey())); + } + } else if (expected instanceof List) { + assertInstanceOf(List.class, actual, "Expected List but got " + actual.getClass()); + List expectedList = (List) expected; + List actualList = (List) actual; + assertEquals(expectedList.size(), actualList.size(), "List size mismatch"); + for (int i = 0; i < expectedList.size(); i++) { + assertDeepEquals(expectedList.get(i), actualList.get(i)); + } + } else if (expected instanceof Number && actual instanceof Number) { + if (expected instanceof Float + || expected instanceof Double + || actual instanceof Float + || actual instanceof Double) { + assertEquals(((Number) expected).doubleValue(), ((Number) actual).doubleValue(), 0.0001); + } else { + assertEquals(((Number) expected).longValue(), ((Number) actual).longValue()); + } + } else { + assertEquals(expected, actual); + } + } + + static class TracesCapture implements ByteBufferConsumer { + int traceCount; + ByteBuffer buffer; + + @Override + public void accept(int messageCount, ByteBuffer buffer) { + this.buffer = buffer; + this.traceCount = messageCount; + } + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeTraceInterceptorTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeTraceInterceptorTest.java new file mode 100644 index 00000000000..d757344634e --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeTraceInterceptorTest.java @@ -0,0 +1,101 @@ +package datadog.trace.common.writer.ddintake; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; +import datadog.trace.common.writer.ListWriter; +import datadog.trace.core.CoreTracer; +import datadog.trace.core.DDCoreJavaSpecification; +import datadog.trace.core.DDSpan; +import java.util.List; +import java.util.concurrent.TimeoutException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.tabletest.junit.TableTest; + +@Timeout(100) +class DDIntakeTraceInterceptorTest extends DDCoreJavaSpecification { + + ListWriter writer; + CoreTracer tracer; + + @BeforeEach + void setup() { + writer = new ListWriter(); + tracer = tracerBuilder().writer(writer).build(); + tracer.addTraceInterceptor(DDIntakeTraceInterceptor.INSTANCE); + } + + @AfterEach + void cleanup() { + if (tracer != null) { + tracer.close(); + } + } + + @TableTest({ + "scenario | httpStatus | expectedHttpStatus", + "null | | ", + "empty string | '' | ", + "string 500 | '500' | 500 ", + "integer 500 | 500 | 500 ", + "integer 600 | 600 | " + }) + void testNormalizationForDdIntake(Object httpStatus, Integer expectedHttpStatus) + throws InterruptedException, TimeoutException { + tracer + .buildSpan("datadog", "my-operation-name") + .withResourceName("my-resource-name") + .withSpanType("my-span-type") + .withServiceName("my-service-name") + .withTag("some-tag-key", "some-tag-value") + .withTag("env", " My_____Env ") + .withTag(Tags.HTTP_STATUS, httpStatus) + .start() + .finish(); + writer.waitForTraces(1); + + List trace = writer.firstTrace(); + assertEquals(1, trace.size()); + DDSpan span = trace.get(0); + assertEquals("my-service-name", span.getServiceName()); + assertEquals("my_operation_name", span.getOperationName().toString()); + assertEquals("my-resource-name", span.getResourceName().toString()); + assertEquals("my-span-type", span.getSpanType()); + assertEquals("some-tag-value", span.getTag("some-tag-key")); + assertEquals("my_env", span.getTag("env")); + assertEquals(expectedHttpStatus, span.getTag(Tags.HTTP_STATUS)); + } + + @Test + void testNormalizationDoesNotImplicitlyConvertSpanType() + throws InterruptedException, TimeoutException { + UTF8BytesString originalSpanType = UTF8BytesString.create("a UTF8 span type"); + tracer + .buildSpan("datadog", "my-operation-name") + .withSpanType(originalSpanType) + .start() + .finish(); + + writer.waitForTraces(1); + + List trace = writer.firstTrace(); + assertEquals(1, trace.size()); + DDSpan span = trace.get(0); + assertEquals(originalSpanType, span.getType()); + } + + @Test + void testDefaultEnvSetting() throws InterruptedException, TimeoutException { + tracer.buildSpan("datadog", "my-operation-name").start().finish(); + writer.waitForTraces(1); + + List trace = writer.firstTrace(); + assertEquals(1, trace.size()); + DDSpan span = trace.get(0); + assertEquals("none", span.getTag("env")); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeTrackTypeResolverTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeTrackTypeResolverTest.java new file mode 100644 index 00000000000..5166a371966 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeTrackTypeResolverTest.java @@ -0,0 +1,30 @@ +package datadog.trace.common.writer.ddintake; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import datadog.trace.api.Config; +import datadog.trace.api.intake.TrackType; +import datadog.trace.test.util.DDJavaSpecification; +import org.tabletest.junit.TableTest; + +class DDIntakeTrackTypeResolverTest extends DDJavaSpecification { + + @TableTest({ + "scenario | ciVisibilityEnabled | ciVisibilityAgentlessEnabled | expectedTrackType", + "ci-vis disabled agentless disabled | false | false | NOOP ", + "ci-vis enabled agentless disabled | true | false | CITESTCYCLE ", + "ci-vis enabled agentless enabled | true | true | CITESTCYCLE " + }) + void shouldReturnTheCorrectTrackType( + boolean ciVisibilityEnabled, + boolean ciVisibilityAgentlessEnabled, + TrackType expectedTrackType) { + Config config = mock(Config.class); + when(config.isCiVisibilityEnabled()).thenReturn(ciVisibilityEnabled); + when(config.isCiVisibilityAgentlessEnabled()).thenReturn(ciVisibilityAgentlessEnabled); + + assertEquals(expectedTrackType, DDIntakeTrackTypeResolver.resolve(config)); + } +} diff --git a/utils/test-junit-converter-utils/src/main/java/datadog/trace/test/junit/utils/tabletest/TableTestTypeConverters.java b/utils/test-junit-converter-utils/src/main/java/datadog/trace/test/junit/utils/tabletest/TableTestTypeConverters.java index 781ca972406..6f4157ab9fe 100644 --- a/utils/test-junit-converter-utils/src/main/java/datadog/trace/test/junit/utils/tabletest/TableTestTypeConverters.java +++ b/utils/test-junit-converter-utils/src/main/java/datadog/trace/test/junit/utils/tabletest/TableTestTypeConverters.java @@ -1,5 +1,7 @@ package datadog.trace.test.junit.utils.tabletest; +import datadog.trace.api.ProtocolVersion; +import datadog.trace.api.intake.TrackType; import org.tabletest.junit.TypeConverter; /** Shared converters for JUnit 5 TableTest tests that use unparsable constants. */ @@ -7,6 +9,30 @@ public final class TableTestTypeConverters { private TableTestTypeConverters() {} + @TypeConverter + public static TrackType toTrackType(String value) { + if (value == null) { + return null; + } + String token = value.trim(); + if (token.startsWith("TrackType.")) { + token = token.substring("TrackType.".length()); + } + return TrackType.valueOf(token); + } + + @TypeConverter + public static ProtocolVersion toProtocolVersion(String value) { + if (value == null) { + return null; + } + String token = value.trim(); + if (token.startsWith("ProtocolVersion.")) { + token = token.substring("ProtocolVersion.".length()); + } + return ProtocolVersion.valueOf(token); + } + @TypeConverter public static long toLong(String value) { if (value == null) { From ca88282d757e6506ac818b969f5c41b0f91936f4 Mon Sep 17 00:00:00 2001 From: jean-philippe bempel Date: Thu, 11 Jun 2026 08:39:43 +0200 Subject: [PATCH 02/10] address comments --- .../common/writer/DDAgentWriterTest.java | 30 +++++--------- .../common/writer/SerializationTest.java | 5 --- .../ddagent/TraceMapperV04PayloadTest.java | 6 +-- .../ddagent/TraceMapperV05PayloadTest.java | 40 ++++++++++--------- 4 files changed, 34 insertions(+), 47 deletions(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterTest.java index 8aeea35d6f2..6a620d35596 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterTest.java @@ -2,6 +2,7 @@ import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.ENQUEUED_FOR_SERIALIZATION; import static datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult.ENQUEUED_FOR_SINGLE_SPAN_SAMPLING; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; @@ -23,11 +24,10 @@ import datadog.trace.core.DDSpan; import datadog.trace.core.monitor.HealthMetrics; import datadog.trace.core.propagation.PropagationTags; +import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.tabletest.junit.TableTest; @@ -37,25 +37,18 @@ class DDAgentWriterTest extends DDCoreJavaSpecification { TraceProcessingWorker worker = mock(TraceProcessingWorker.class); DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); DDAgentApi api = mock(DDAgentApi.class); - MonitoringImpl monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS); + MonitoringImpl monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, SECONDS); PayloadDispatcherImpl dispatcher = new PayloadDispatcherImpl(new DDAgentMapperDiscovery(discovery), api, monitor, monitoring); - DDAgentWriter writer = new DDAgentWriter(worker, dispatcher, monitor, 1, TimeUnit.SECONDS, false); + DDAgentWriter writer = new DDAgentWriter(worker, dispatcher, monitor, 1, SECONDS, false); // Only used to create spans - CoreTracer dummyTracer; - - @BeforeEach - void setup() { - dummyTracer = tracerBuilder().writer(new ListWriter()).build(); - } + CoreTracer dummyTracer = tracerBuilder().writer(new ListWriter()).build(); @AfterEach void cleanup() { writer.close(); - if (dummyTracer != null) { - dummyTracer.close(); - } + dummyTracer.close(); } @Test @@ -91,13 +84,13 @@ void testWriterStartClosed() { @Test void testWriterFlush() { - when(worker.flush(1, TimeUnit.SECONDS)).thenReturn(true, false); + when(worker.flush(1, SECONDS)).thenReturn(true, false); // first flush succeeds writer.flush(); // monitor is notified - verify(worker).flush(1, TimeUnit.SECONDS); + verify(worker).flush(1, SECONDS); verify(monitor).onFlush(false); verifyNoMoreInteractions(monitor, worker, discovery, api); @@ -107,7 +100,7 @@ void testWriterFlush() { writer.flush(); // no additional monitor notifications - verify(worker).flush(1, TimeUnit.SECONDS); + verify(worker).flush(1, SECONDS); verifyNoMoreInteractions(monitor, worker, discovery, api); } @@ -207,10 +200,9 @@ void testDroppedTraceIsCounted(PublishResult publishResult) { HealthMetrics localMonitor = mock(HealthMetrics.class); PayloadDispatcherImpl localDispatcher = mock(PayloadDispatcherImpl.class); DDAgentWriter localWriter = - new DDAgentWriter(localWorker, localDispatcher, localMonitor, 1, TimeUnit.SECONDS, false); + new DDAgentWriter(localWorker, localDispatcher, localMonitor, 1, SECONDS, false); - DDSpan p0 = newSpan(); - List trace = java.util.Arrays.asList(p0, newSpan()); + List trace = Arrays.asList(newSpan(), newSpan()); when(localWorker.publish(eq(trace.get(0)), anyInt(), eq(trace))).thenReturn(publishResult); localWriter.write(trace); diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/SerializationTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/SerializationTest.java index a0254db18d0..0924581f26b 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/SerializationTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/SerializationTest.java @@ -19,17 +19,14 @@ class SerializationTest extends DDJavaSpecification { @Test void testJsonMapperSerialization() throws Exception { - // setup ObjectMapper mapper = new ObjectMapper(); Map map = singletonMap("key1", "val1"); byte[] serializedMap = mapper.writeValueAsBytes(map); byte[] serializedList = ("[" + new String(serializedMap) + "]").getBytes(); - // when List> result = mapper.readValue(serializedList, new TypeReference>>() {}); - // then assertEquals(Collections.singletonList(map), result); assertEquals("[{\"key1\":\"val1\"}]", new String(serializedList)); } @@ -55,11 +52,9 @@ void testMsgpackMapperSerialization() throws Exception { } byte[] serializedList = packer.toByteArray(); - // when List> result = mapper.readValue(serializedList, new TypeReference>>() {}); - // then assertEquals(input, result); } } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java index 87496e430df..8bab7bd1cc2 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java @@ -442,10 +442,6 @@ public void close() {} } private static void assertEqualsWithNullAsEmpty(CharSequence expected, CharSequence actual) { - if (null == expected) { - assertEquals("", actual); - } else { - assertEquals(expected.toString(), actual.toString()); - } + assertEquals(expected == null ? "" : expected.toString(), actual.toString()); } } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV05PayloadTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV05PayloadTest.java index 453a35f04bc..b35ea8fa8c3 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV05PayloadTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV05PayloadTest.java @@ -53,8 +53,7 @@ void resetProcessTags() { @Test void testBodyOverflowCausesFlush() { // disable process tags since they are only on the first span of the chunk otherwise the - // calculation woes - // 4x 36 ASCII characters and 2 bytes of msgpack string prefix + // calculation woes 4x 36 ASCII characters and 2 bytes of msgpack string prefix int dictionarySpacePerTrace = 4 * (36 + 2); // enough space for two traces with distinct string values, plus the header int dictionarySize = dictionarySpacePerTrace * 2 + 5; @@ -263,22 +262,27 @@ public void accept(int messageCount, ByteBuffer buffer) { meta.put(dictionary[unpacker.unpackInt()], dictionary[unpacker.unpackInt()]); } for (Map.Entry entry : meta.entrySet()) { - if (Tags.HTTP_STATUS.equals(entry.getKey())) { - assertEquals(String.valueOf(expectedSpan.getHttpStatusCode()), entry.getValue()); - } else if (DDTags.ORIGIN_KEY.equals(entry.getKey())) { - assertEquals(expectedSpan.getOrigin(), entry.getValue()); - } else if (DDTags.PROCESS_TAGS.equals(entry.getKey())) { - processTagsCount++; - assertTrue(Config.get().isExperimentalPropagateProcessTagsEnabled()); - assertEquals(0, k); - assertEquals(ProcessTags.getTagsForSerialization().toString(), entry.getValue()); - } else { - Object tag = expectedSpan.getTag(entry.getKey()); - if (null != tag) { - assertEquals(String.valueOf(tag), entry.getValue()); - } else { - assertEquals(expectedSpan.getBaggage().get(entry.getKey()), entry.getValue()); - } + switch (entry.getKey()) { + case Tags.HTTP_STATUS: + assertEquals(String.valueOf(expectedSpan.getHttpStatusCode()), entry.getValue()); + break; + case DDTags.ORIGIN_KEY: + assertEquals(expectedSpan.getOrigin(), entry.getValue()); + break; + case DDTags.PROCESS_TAGS: + processTagsCount++; + assertTrue(Config.get().isExperimentalPropagateProcessTagsEnabled()); + assertEquals(0, k); + assertEquals(ProcessTags.getTagsForSerialization().toString(), entry.getValue()); + break; + default: + Object tag = expectedSpan.getTag(entry.getKey()); + if (null != tag) { + assertEquals(String.valueOf(tag), entry.getValue()); + } else { + assertEquals(expectedSpan.getBaggage().get(entry.getKey()), entry.getValue()); + } + break; } } int metricsSize = unpacker.unpackMapHeader(); From bd6af4948f7f7249c81d3749328be52ae0a2ac19 Mon Sep 17 00:00:00 2001 From: jean-philippe bempel Date: Thu, 11 Jun 2026 16:44:43 +0200 Subject: [PATCH 03/10] fix asserts --- .../datadog/trace/common/writer/DDAgentApiTest.java | 4 +++- .../trace/common/writer/DDAgentWriterTest.java | 11 ++++++++--- .../trace/common/writer/ddintake/DDIntakeApiTest.java | 5 ++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java index 1452c640509..b759ea525f2 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java @@ -2,7 +2,9 @@ import static datadog.trace.api.ProtocolVersion.V0_5; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -382,7 +384,7 @@ void testApiResponseListenersSee200Responses(String agentVersion) { client.sendSerializedTraces(payload); Map> response = agentResponse.get(); - assertTrue(response != null && response.containsKey("hello")); + assertEquals(singletonMap("hello", emptyMap()), response); assertEquals("java", agent.getLastRequest().getHeaders().get("Datadog-Meta-Lang")); assertEquals( System.getProperty("java.version", "unknown"), diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterTest.java index 6a620d35596..873f80f9a48 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterTest.java @@ -16,6 +16,7 @@ import datadog.communication.ddagent.DDAgentFeaturesDiscovery; import datadog.metrics.api.statsd.StatsDClient; import datadog.metrics.impl.MonitoringImpl; +import datadog.trace.api.sampling.PrioritySampling; import datadog.trace.common.writer.ddagent.DDAgentApi; import datadog.trace.common.writer.ddagent.DDAgentMapperDiscovery; import datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult; @@ -202,12 +203,16 @@ void testDroppedTraceIsCounted(PublishResult publishResult) { DDAgentWriter localWriter = new DDAgentWriter(localWorker, localDispatcher, localMonitor, 1, SECONDS, false); - List trace = Arrays.asList(newSpan(), newSpan()); + DDSpan p0 = newSpan(); + p0.setSamplingPriority(PrioritySampling.SAMPLER_DROP); + List trace = Arrays.asList(p0, newSpan()); - when(localWorker.publish(eq(trace.get(0)), anyInt(), eq(trace))).thenReturn(publishResult); + when(localWorker.publish(eq(trace.get(0)), eq((int) PrioritySampling.SAMPLER_DROP), eq(trace))) + .thenReturn(publishResult); localWriter.write(trace); - verify(localWorker).publish(eq(trace.get(0)), anyInt(), eq(trace)); + verify(localWorker) + .publish(eq(trace.get(0)), eq((int) PrioritySampling.SAMPLER_DROP), eq(trace)); verify(localDispatcher).onDroppedTrace(trace.size()); } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeApiTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeApiTest.java index 4609c585a65..dfc0386a2ec 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeApiTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeApiTest.java @@ -32,6 +32,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.OptionalInt; import java.util.TreeMap; import java.util.zip.GZIPInputStream; import okhttp3.HttpUrl; @@ -327,7 +328,9 @@ private void runContentIsSentAsMsgpackTest( Payload payload = prepareTraces(trackType, traces); try { - client.sendSerializedTraces(payload).status(); + OptionalInt status = client.sendSerializedTraces(payload).status(); + assertTrue(status.isPresent()); + assertEquals(200, status.getAsInt()); assertEquals("application/msgpack", intake.getLastRequest().getContentType()); Map actualBody = convertMap(intake.getLastRequest().getBody()); assertDeepEquals(expectedRequestBody, actualBody); From f882d995fca3fe38893918bfd88fd708b811ba55 Mon Sep 17 00:00:00 2001 From: jean-philippe bempel Date: Fri, 12 Jun 2026 09:56:31 +0200 Subject: [PATCH 04/10] fix comments --- .../writer/DDIntakeWriterCombinedTest.java | 3 +- .../writer/TraceProcessingWorkerTest.java | 82 +++++++------------ 2 files changed, 30 insertions(+), 55 deletions(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterCombinedTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterCombinedTest.java index 56883149858..c53286cb106 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterCombinedTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterCombinedTest.java @@ -40,6 +40,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -322,7 +323,7 @@ void monitorHappyPath() { server.handlers(h -> h.post(path, api -> api.getResponse().status(200).send()))); try { HttpUrl hostUrl = HttpUrl.get(intake.getAddress()); - okhttp3.OkHttpClient httpClient = OkHttpUtils.buildHttpClient(hostUrl, 1000); + OkHttpClient httpClient = OkHttpUtils.buildHttpClient(hostUrl, 1000); DDIntakeApi api = DDIntakeApi.builder() .hostUrl(hostUrl) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java index 6f57f5d455b..bc51e4effd9 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java @@ -92,7 +92,7 @@ void testHeartbeatsShouldBeTriggeredAutomaticallyWhenEnabled() throws Exception @Test void testHeartbeatsShouldOccurAtLeastOncePerSecondWhenNotThrottled() throws Exception { AtomicInteger flushCount = new AtomicInteger(); - TraceProcessingWorker worker = + try (TraceProcessingWorker worker = new TraceProcessingWorker( 10, mock(HealthMetrics.class), @@ -101,12 +101,11 @@ void testHeartbeatsShouldOccurAtLeastOncePerSecondWhenNotThrottled() throws Exce FAST_LANE, 1, TimeUnit.NANOSECONDS, // stop heartbeats from being throttled - null); + null)) { - // processor is started - worker.start(); + // processor is started + worker.start(); - try { // heartbeat occurs automatically approximately once per second // wait 1 second initial delay, then poll up to 1 second Thread.sleep(1000); @@ -116,9 +115,6 @@ void testHeartbeatsShouldOccurAtLeastOncePerSecondWhenNotThrottled() throws Exce Thread.sleep(50); } assertTrue(flushCount.get() > 1); - } finally { - // cleanup - worker.close(); } } @@ -129,7 +125,9 @@ void testHeartbeatsShouldOccurAtLeastOncePerSecondWhenNotThrottled() throws Exce @Test void testAFlushShouldClearThePrimaryQueue() { AtomicInteger flushCount = new AtomicInteger(); - TraceProcessingWorker worker = + // prevent heartbeats from helping the flush happen + + try (TraceProcessingWorker worker = new TraceProcessingWorker( 10, mock(HealthMetrics.class), @@ -138,9 +136,7 @@ void testAFlushShouldClearThePrimaryQueue() { FAST_LANE, 100, TimeUnit.SECONDS, // prevent heartbeats from helping the flush happen - null); - - try { + null)) { // there is pending work it is completed before a flush // processing this span will throw an exception, but it should be caught // and not disrupt the flush @@ -153,9 +149,6 @@ void testAFlushShouldClearThePrimaryQueue() { assertTrue(flushed); assertEquals(1, flushCount.get()); assertTrue(worker.getPrimaryQueue().isEmpty()); - } finally { - // cleanup - worker.close(); } } @@ -195,7 +188,9 @@ void testShouldReportFailureIfSerializationFails( .when(healthMetrics) .onFailedSerialize(any(), any()); - TraceProcessingWorker worker = + // prevent heartbeats from helping the flush happen + + try (TraceProcessingWorker worker = new TraceProcessingWorker( 10, healthMetrics, @@ -204,10 +199,8 @@ void testShouldReportFailureIfSerializationFails( FAST_LANE, 100, TimeUnit.SECONDS, // prevent heartbeats from helping the flush happen - null); - worker.start(); - - try { + null)) { + worker.start(); // a trace is processed but can't be passed on worker.publish(mock(DDSpan.class), priority, Collections.singletonList(mock(DDSpan.class))); @@ -218,9 +211,6 @@ void testShouldReportFailureIfSerializationFails( Thread.sleep(50); } assertEquals(1, errorReported.get()); - } finally { - // cleanup - worker.close(); } } @@ -276,7 +266,7 @@ void testTraceShouldBePostProcessed() throws Exception { SpanPostProcessor.Holder.INSTANCE = mockProcessor; - TraceProcessingWorker worker = + try (TraceProcessingWorker worker = new TraceProcessingWorker( 10, healthMetrics, @@ -285,10 +275,8 @@ void testTraceShouldBePostProcessed() throws Exception { FAST_LANE, 100, TimeUnit.SECONDS, - null); - worker.start(); - - try { + null)) { + worker.start(); // traces are submitted List trace = new ArrayList<>(); trace.add(span1); @@ -305,9 +293,7 @@ void testTraceShouldBePostProcessed() throws Exception { assertTrue(processedSpan1.get()); assertTrue(processedSpan2.get()); } finally { - // cleanup SpanPostProcessor.Holder.INSTANCE = SpanPostProcessor.Holder.NOOP; - worker.close(); } } @@ -350,7 +336,9 @@ void testTracesShouldBeProcessed( .when(countingDispatcher) .addTrace(any()); HealthMetrics healthMetrics = mock(HealthMetrics.class); - TraceProcessingWorker worker = + // prevent heartbeats from helping the flush happen + + try (TraceProcessingWorker worker = new TraceProcessingWorker( 10, healthMetrics, @@ -359,10 +347,8 @@ void testTracesShouldBeProcessed( FAST_LANE, 100, TimeUnit.SECONDS, // prevent heartbeats from helping the flush happen - null); - worker.start(); - - try { + null)) { + worker.start(); // traces are submitted int submitted = 0; for (int i = 0; i < traceCount; ++i) { @@ -380,10 +366,8 @@ void testTracesShouldBeProcessed( Thread.sleep(50); } assertEquals(expectedSubmitted, acceptedCount.get()); - } finally { - // cleanup - worker.close(); } + // cleanup } // ------------------------------------------------------------------------- @@ -490,7 +474,7 @@ public > boolean setSamplingPriority(T span) { } }; - TraceProcessingWorker worker = + try (TraceProcessingWorker worker = new TraceProcessingWorker( 10, healthMetrics, @@ -499,10 +483,8 @@ public > boolean setSamplingPriority(T span) { FAST_LANE, 100, TimeUnit.SECONDS, - singleSpanSampler); - worker.start(); - - try { + singleSpanSampler)) { + worker.start(); // traces are submitted for (int i = 0; i < traceCount; ++i) { List trace = new ArrayList<>(); @@ -523,9 +505,6 @@ public > boolean setSamplingPriority(T span) { assertEquals(acceptedTraces, acceptedCount.get()); assertEquals(acceptedSpans, acceptedSpanCount.get()); assertEquals(sampledSingleSpans, sampledSpansCount.get()); - } finally { - // cleanup - worker.close(); } } @@ -604,7 +583,7 @@ public > boolean setSamplingPriority(T span) { } }; - TraceProcessingWorker worker = + try (TraceProcessingWorker worker = new TraceProcessingWorker( 10, healthMetrics, @@ -613,10 +592,8 @@ public > boolean setSamplingPriority(T span) { FAST_LANE, 100, TimeUnit.SECONDS, - singleSpanSampler); - worker.start(); - - try { + singleSpanSampler)) { + worker.start(); // traces are submitted for (int i = 0; i < traceCount; ++i) { List trace = new ArrayList<>(); @@ -637,9 +614,6 @@ public > boolean setSamplingPriority(T span) { assertEquals(expectedChunks, chunksCount.get()); assertEquals(expectedSpans, spansCount.get()); assertEquals(sampledSingleSpans, sampledSpansCount.get()); - } finally { - // cleanup - worker.close(); } } } From caf4d1af80321790798d61ad3f3196ef00e9a727 Mon Sep 17 00:00:00 2001 From: jean-philippe bempel Date: Wed, 17 Jun 2026 13:12:53 +0200 Subject: [PATCH 05/10] address comments --- .../ddagent/TraceMapperV04PayloadTest.java | 255 +++++++++--------- 1 file changed, 128 insertions(+), 127 deletions(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java index 8bab7bd1cc2..5887565208b 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java @@ -265,140 +265,141 @@ public void accept(int messageCount, ByteBuffer buffer) { Payload payload = mapper.newPayload().withBody(messageCount, buffer); payload.writeTo(this); captured.flip(); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(captured); - int traceCount = unpacker.unpackArrayHeader(); - for (int i = 0; i < traceCount; ++i) { - List expectedTrace = expectedTraces.get(position++); - int spanCount = unpacker.unpackArrayHeader(); - assertEquals(expectedTrace.size(), spanCount); - for (int k = 0; k < spanCount; ++k) { - TraceGenerator.PojoSpan expectedSpan = expectedTrace.get(k); - int elementCount = unpacker.unpackMapHeader(); - boolean hasMetaStruct = !expectedSpan.getMetaStruct().isEmpty(); - assertEquals(hasMetaStruct ? 13 : 12, elementCount); - assertEquals("service", unpacker.unpackString()); - String serviceName = unpacker.unpackString(); - assertEqualsWithNullAsEmpty(expectedSpan.getServiceName(), serviceName); - assertEquals("name", unpacker.unpackString()); - String operationName = unpacker.unpackString(); - assertEqualsWithNullAsEmpty(expectedSpan.getOperationName(), operationName); - assertEquals("resource", unpacker.unpackString()); - String resourceName = unpacker.unpackString(); - assertEqualsWithNullAsEmpty(expectedSpan.getResourceName(), resourceName); - assertEquals("trace_id", unpacker.unpackString()); - long traceId = unpacker.unpackValue().asNumberValue().toLong(); - assertEquals(expectedSpan.getTraceId().toLong(), traceId); - assertEquals("span_id", unpacker.unpackString()); - long spanId = unpacker.unpackValue().asNumberValue().toLong(); - assertEquals(expectedSpan.getSpanId(), spanId); - assertEquals("parent_id", unpacker.unpackString()); - long parentId = unpacker.unpackValue().asNumberValue().toLong(); - assertEquals(expectedSpan.getParentId(), parentId); - assertEquals("start", unpacker.unpackString()); - long startTime = unpacker.unpackLong(); - assertEquals(expectedSpan.getStartTime(), startTime); - assertEquals("duration", unpacker.unpackString()); - long duration = unpacker.unpackLong(); - assertEquals(expectedSpan.getDurationNano(), duration); - assertEquals("type", unpacker.unpackString()); - String type = unpacker.unpackString(); - assertEquals(expectedSpan.getType(), type); - assertEquals("error", unpacker.unpackString()); - int error = unpacker.unpackInt(); - assertEquals(expectedSpan.getError(), error); - assertEquals("metrics", unpacker.unpackString()); - int metricsSize = unpacker.unpackMapHeader(); - HashMap metrics = new HashMap<>(); - for (int j = 0; j < metricsSize; ++j) { - String key = unpacker.unpackString(); - Number metricValue = null; - MessageFormat format = unpacker.getNextFormat(); - switch (format) { - case NEGFIXINT: - case POSFIXINT: - case INT8: - case UINT8: - case INT16: - case UINT16: - case INT32: - case UINT32: - metricValue = unpacker.unpackInt(); - break; - case INT64: - case UINT64: - metricValue = unpacker.unpackLong(); - break; - case FLOAT32: - metricValue = unpacker.unpackFloat(); - break; - case FLOAT64: - metricValue = unpacker.unpackDouble(); - break; - default: - Assertions.fail("Unexpected type in metrics values: " + format); - } - if (DD_MEASURED.toString().equals(key)) { - assertTrue(metricValue.intValue() == 1 || !expectedSpan.isMeasured()); - } else if (DDSpanContext.PRIORITY_SAMPLING_KEY.equals(key)) { - // check that priority sampling is only on first and last span - if (k == 0 || k == spanCount - 1) { - assertEquals(expectedSpan.samplingPriority(), metricValue.intValue()); + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(captured)) { + int traceCount = unpacker.unpackArrayHeader(); + for (int i = 0; i < traceCount; ++i) { + List expectedTrace = expectedTraces.get(position++); + int spanCount = unpacker.unpackArrayHeader(); + assertEquals(expectedTrace.size(), spanCount); + for (int k = 0; k < spanCount; ++k) { + TraceGenerator.PojoSpan expectedSpan = expectedTrace.get(k); + int elementCount = unpacker.unpackMapHeader(); + boolean hasMetaStruct = !expectedSpan.getMetaStruct().isEmpty(); + assertEquals(hasMetaStruct ? 13 : 12, elementCount); + assertEquals("service", unpacker.unpackString()); + String serviceName = unpacker.unpackString(); + assertEqualsWithNullAsEmpty(expectedSpan.getServiceName(), serviceName); + assertEquals("name", unpacker.unpackString()); + String operationName = unpacker.unpackString(); + assertEqualsWithNullAsEmpty(expectedSpan.getOperationName(), operationName); + assertEquals("resource", unpacker.unpackString()); + String resourceName = unpacker.unpackString(); + assertEqualsWithNullAsEmpty(expectedSpan.getResourceName(), resourceName); + assertEquals("trace_id", unpacker.unpackString()); + long traceId = unpacker.unpackValue().asNumberValue().toLong(); + assertEquals(expectedSpan.getTraceId().toLong(), traceId); + assertEquals("span_id", unpacker.unpackString()); + long spanId = unpacker.unpackValue().asNumberValue().toLong(); + assertEquals(expectedSpan.getSpanId(), spanId); + assertEquals("parent_id", unpacker.unpackString()); + long parentId = unpacker.unpackValue().asNumberValue().toLong(); + assertEquals(expectedSpan.getParentId(), parentId); + assertEquals("start", unpacker.unpackString()); + long startTime = unpacker.unpackLong(); + assertEquals(expectedSpan.getStartTime(), startTime); + assertEquals("duration", unpacker.unpackString()); + long duration = unpacker.unpackLong(); + assertEquals(expectedSpan.getDurationNano(), duration); + assertEquals("type", unpacker.unpackString()); + String type = unpacker.unpackString(); + assertEquals(expectedSpan.getType(), type); + assertEquals("error", unpacker.unpackString()); + int error = unpacker.unpackInt(); + assertEquals(expectedSpan.getError(), error); + assertEquals("metrics", unpacker.unpackString()); + int metricsSize = unpacker.unpackMapHeader(); + HashMap metrics = new HashMap<>(); + for (int j = 0; j < metricsSize; ++j) { + String key = unpacker.unpackString(); + Number metricValue = null; + MessageFormat format = unpacker.getNextFormat(); + switch (format) { + case NEGFIXINT: + case POSFIXINT: + case INT8: + case UINT8: + case INT16: + case UINT16: + case INT32: + case UINT32: + metricValue = unpacker.unpackInt(); + break; + case INT64: + case UINT64: + metricValue = unpacker.unpackLong(); + break; + case FLOAT32: + metricValue = unpacker.unpackFloat(); + break; + case FLOAT64: + metricValue = unpacker.unpackDouble(); + break; + default: + Assertions.fail("Unexpected type in metrics values: " + format); + } + if (DD_MEASURED.toString().equals(key)) { + assertTrue(metricValue.intValue() == 1 || !expectedSpan.isMeasured()); + } else if (DDSpanContext.PRIORITY_SAMPLING_KEY.equals(key)) { + // check that priority sampling is only on first and last span + if (k == 0 || k == spanCount - 1) { + assertEquals(expectedSpan.samplingPriority(), metricValue.intValue()); + } else { + assertFalse(expectedSpan.hasSamplingPriority()); + } } else { - assertFalse(expectedSpan.hasSamplingPriority()); + metrics.put(key, metricValue); } - } else { - metrics.put(key, metricValue); } - } - for (Map.Entry metric : metrics.entrySet()) { - if (metric.getValue() instanceof Double || metric.getValue() instanceof Float) { - assertEquals( - ((Number) expectedSpan.getTag(metric.getKey())).doubleValue(), - metric.getValue().doubleValue(), - 0.001); - } else { - // Groovy compared numerically, Java requires explicit long comparison to avoid - // Long/Integer type mismatch from different msgpack integer encoding widths - assertEquals( - ((Number) expectedSpan.getTag(metric.getKey())).longValue(), - metric.getValue().longValue()); + metrics.forEach((key, value) -> { + if (value instanceof Double || value instanceof Float) { + assertEquals( + ((Number) expectedSpan.getTag(key)).doubleValue(), + value.doubleValue(), + 0.001); + } else { + // Groovy compared numerically, Java requires explicit long comparison to avoid + // Long/Integer type mismatch from different msgpack integer encoding widths + assertEquals( + ((Number) expectedSpan.getTag(key)).longValue(), + value.longValue()); + } + }); + assertEquals("meta", unpacker.unpackString()); + int metaSize = unpacker.unpackMapHeader(); + HashMap meta = new HashMap<>(); + for (int j = 0; j < metaSize; ++j) { + meta.put(unpacker.unpackString(), unpacker.unpackString()); } - } - assertEquals("meta", unpacker.unpackString()); - int metaSize = unpacker.unpackMapHeader(); - HashMap meta = new HashMap<>(); - for (int j = 0; j < metaSize; ++j) { - meta.put(unpacker.unpackString(), unpacker.unpackString()); - } - for (Map.Entry entry : meta.entrySet()) { - if (Tags.HTTP_STATUS.equals(entry.getKey())) { - assertEquals(String.valueOf(expectedSpan.getHttpStatusCode()), entry.getValue()); - } else if (DDTags.ORIGIN_KEY.equals(entry.getKey())) { - assertEquals(expectedSpan.getOrigin(), entry.getValue()); - } else if (DDTags.PROCESS_TAGS.equals(entry.getKey())) { - assertTrue(Config.get().isExperimentalPropagateProcessTagsEnabled()); - assertEquals(0, k); - assertEquals(ProcessTags.getTagsForSerialization().toString(), entry.getValue()); - processTagsCount++; - } else { - Object tag = expectedSpan.getTag(entry.getKey()); - if (null != tag) { - assertEquals(String.valueOf(tag), entry.getValue()); + for (Map.Entry entry : meta.entrySet()) { + if (Tags.HTTP_STATUS.equals(entry.getKey())) { + assertEquals(String.valueOf(expectedSpan.getHttpStatusCode()), entry.getValue()); + } else if (DDTags.ORIGIN_KEY.equals(entry.getKey())) { + assertEquals(expectedSpan.getOrigin(), entry.getValue()); + } else if (DDTags.PROCESS_TAGS.equals(entry.getKey())) { + assertTrue(Config.get().isExperimentalPropagateProcessTagsEnabled()); + assertEquals(0, k); + assertEquals(ProcessTags.getTagsForSerialization().toString(), entry.getValue()); + processTagsCount++; } else { - assertEquals(expectedSpan.getBaggage().get(entry.getKey()), entry.getValue()); + Object tag = expectedSpan.getTag(entry.getKey()); + if (null != tag) { + assertEquals(String.valueOf(tag), entry.getValue()); + } else { + assertEquals(expectedSpan.getBaggage().get(entry.getKey()), entry.getValue()); + } } } - } - if (hasMetaStruct) { - Map metaStruct = expectedSpan.getMetaStruct(); - assertEquals("meta_struct", unpacker.unpackString()); - int metaStructSize = unpacker.unpackMapHeader(); - for (int j = 0; j < metaStructSize; ++j) { - String field = unpacker.unpackString(); - if (metaStructVerifier != null) { - byte[] binary = new byte[unpacker.unpackBinaryHeader()]; - unpacker.readPayload(binary); - metaStructVerifier.verify(metaStruct.get(field), binary); + if (hasMetaStruct) { + Map metaStruct = expectedSpan.getMetaStruct(); + assertEquals("meta_struct", unpacker.unpackString()); + int metaStructSize = unpacker.unpackMapHeader(); + for (int j = 0; j < metaStructSize; ++j) { + String field = unpacker.unpackString(); + if (metaStructVerifier != null) { + byte[] binary = new byte[unpacker.unpackBinaryHeader()]; + unpacker.readPayload(binary); + metaStructVerifier.verify(metaStruct.get(field), binary); + } } } } From 3d53eb76ac0684bcc98dbc93bffb682a89c80ee6 Mon Sep 17 00:00:00 2001 From: jean-philippe bempel Date: Mon, 6 Jul 2026 17:44:20 +0200 Subject: [PATCH 06/10] rebase + use new converters spotless --- .../common/writer/PrioritizationTest.java | 2 +- .../trace/common/writer/TraceMapperTest.java | 2 +- .../writer/TraceProcessingWorkerTest.java | 2 +- .../ddagent/TraceMapperV04PayloadTest.java | 29 ++++++++++--------- .../ddagent/TraceMapperV1PayloadTest.java | 19 ++++-------- .../AbstractClassConstantConvertor.java | 13 +++++++-- .../converter/SamplingMechanismConverter.java | 10 +++++++ 7 files changed, 44 insertions(+), 33 deletions(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/PrioritizationTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/PrioritizationTest.java index 016f0da6198..4bf08ab36d6 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/PrioritizationTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/PrioritizationTest.java @@ -17,7 +17,7 @@ import datadog.trace.common.writer.ddagent.PrioritizationStrategy; import datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult; import datadog.trace.core.DDSpan; -import datadog.trace.junit.utils.tabletest.PrioritySamplingConverter; +import datadog.trace.junit.utils.converter.PrioritySamplingConverter; import datadog.trace.test.util.DDJavaSpecification; import java.util.Collections; import java.util.List; diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceMapperTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceMapperTest.java index 2d28a395412..2cf4129b664 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceMapperTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceMapperTest.java @@ -35,7 +35,7 @@ void testTraceMapperV05() throws Exception { .withTag("elasticsearch.version", "7.0") .start(); span.setBaggageItem("baggage", "item"); - span.context().setDataTop("mydata", "[1,2,3]"); + span.spanContext().setDataTop("mydata", "[1,2,3]"); List trace = Collections.singletonList(span); TraceMapperV0_5 traceMapper = new TraceMapperV0_5(); diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java index bc51e4effd9..63fe347739c 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java @@ -19,7 +19,7 @@ import datadog.trace.core.DDSpanContext; import datadog.trace.core.PendingTrace; import datadog.trace.core.monitor.HealthMetrics; -import datadog.trace.junit.utils.tabletest.PrioritySamplingConverter; +import datadog.trace.junit.utils.converter.PrioritySamplingConverter; import datadog.trace.test.util.DDJavaSpecification; import java.lang.reflect.Method; import java.util.ArrayList; diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java index 5887565208b..aa290d652ea 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java @@ -350,20 +350,21 @@ public void accept(int messageCount, ByteBuffer buffer) { metrics.put(key, metricValue); } } - metrics.forEach((key, value) -> { - if (value instanceof Double || value instanceof Float) { - assertEquals( - ((Number) expectedSpan.getTag(key)).doubleValue(), - value.doubleValue(), - 0.001); - } else { - // Groovy compared numerically, Java requires explicit long comparison to avoid - // Long/Integer type mismatch from different msgpack integer encoding widths - assertEquals( - ((Number) expectedSpan.getTag(key)).longValue(), - value.longValue()); - } - }); + metrics.forEach( + (key, value) -> { + if (value instanceof Double || value instanceof Float) { + assertEquals( + ((Number) expectedSpan.getTag(key)).doubleValue(), + value.doubleValue(), + 0.001); + } else { + // Groovy compared numerically, Java requires explicit long comparison to + // avoid + // Long/Integer type mismatch from different msgpack integer encoding widths + assertEquals( + ((Number) expectedSpan.getTag(key)).longValue(), value.longValue()); + } + }); assertEquals("meta", unpacker.unpackString()); int metaSize = unpacker.unpackMapHeader(); HashMap meta = new HashMap<>(); diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java index 2b5ea0bf7c7..2f4fd225124 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java @@ -29,7 +29,7 @@ import datadog.trace.common.writer.Payload; import datadog.trace.common.writer.TraceGenerator; import datadog.trace.core.MetadataConsumer; -import datadog.trace.junit.utils.tabletest.SamplingMechanismConverter; +import datadog.trace.junit.utils.converter.SamplingMechanismConverter; import datadog.trace.test.util.DDJavaSpecification; import java.io.IOException; import java.nio.ByteBuffer; @@ -194,11 +194,11 @@ void testPayloadContainsExpectedHeaderAndChunkFields() throws IOException { } @TableTest({ - "scenario | decisionMakerTag | expectedSamplingMechanism", - "null tag | | SamplingMechanism.DEFAULT", - "simple negative | '-3' | 3 ", - "compound | '934086a686-7' | 7 ", - "invalid | 'invalid' | SamplingMechanism.DEFAULT" + "scenario | decisionMakerTag | expectedSamplingMechanism ", + "null tag | | SamplingMechanism.DEFAULT ", + "simple negative | '-3' | SamplingMechanism.LOCAL_USER_RULE", + "compound | '934086a686-7' | 7 ", + "invalid | 'invalid' | SamplingMechanism.DEFAULT " }) void testSamplingMechanismNormalizationFromDdPDm( String decisionMakerTag, @@ -937,13 +937,6 @@ public void processTagsAndBaggage(MetadataConsumer consumer) { processTagsAndBaggageCount++; super.processTagsAndBaggage(consumer); } - - @Override - public void processTagsAndBaggage( - MetadataConsumer consumer, boolean injectLinksAsTags, boolean injectBaggageAsTags) { - processTagsAndBaggageCount++; - super.processTagsAndBaggage(consumer, injectLinksAsTags, injectBaggageAsTags); - } } private static class ByteArrayChannel implements WritableByteChannel { diff --git a/utils/test-junit-converter-utils/src/main/java/datadog/trace/test/junit/utils/converter/AbstractClassConstantConvertor.java b/utils/test-junit-converter-utils/src/main/java/datadog/trace/test/junit/utils/converter/AbstractClassConstantConvertor.java index 122a9d9ad52..f68d4f813c0 100644 --- a/utils/test-junit-converter-utils/src/main/java/datadog/trace/test/junit/utils/converter/AbstractClassConstantConvertor.java +++ b/utils/test-junit-converter-utils/src/main/java/datadog/trace/test/junit/utils/converter/AbstractClassConstantConvertor.java @@ -32,9 +32,12 @@ public T convert(Object source, ParameterContext context) { s = s.substring(length + 1); } T mappedValue = mapping().get(s); - if (mappedValue == null && throwsOnUnsupportedValue()) { - throw new ArgumentConversionException( - "Unsupported constant " + source + " from " + className); + if (mappedValue == null) { + if (throwsOnUnsupportedValue()) { + throw new ArgumentConversionException( + "Unsupported constant " + source + " from " + className); + } + return convertWhenNoMapping(source); } return mappedValue; } @@ -43,6 +46,10 @@ protected boolean throwsOnUnsupportedValue() { return true; } + protected T convertWhenNoMapping(Object source) { + return null; + } + public abstract static class AbstractStringFallThruConverter extends AbstractClassConstantConvertor { @Override diff --git a/utils/test-junit-converter-utils/src/main/java/datadog/trace/test/junit/utils/converter/SamplingMechanismConverter.java b/utils/test-junit-converter-utils/src/main/java/datadog/trace/test/junit/utils/converter/SamplingMechanismConverter.java index 044edabead9..2b30a4e82dd 100644 --- a/utils/test-junit-converter-utils/src/main/java/datadog/trace/test/junit/utils/converter/SamplingMechanismConverter.java +++ b/utils/test-junit-converter-utils/src/main/java/datadog/trace/test/junit/utils/converter/SamplingMechanismConverter.java @@ -48,4 +48,14 @@ protected String className() { protected Map mapping() { return MAPPING; } + + @Override + protected boolean throwsOnUnsupportedValue() { + return false; + } + + @Override + protected Byte convertWhenNoMapping(Object source) { + return Byte.parseByte((String) source); + } } From e120ab9d5a1f8f1a0a9f7c182cee6ece0054a6aa Mon Sep 17 00:00:00 2001 From: Jean-Philippe Bempel Date: Wed, 19 Aug 2026 16:48:26 +0200 Subject: [PATCH 07/10] rebase --- .../trace/common/writer/DDAgentApiTest.java | 49 +- .../writer/DDAgentWriterCombinedTest.java | 2 +- .../common/writer/PrioritizationTest.java | 2 +- .../writer/TraceProcessingWorkerTest.java | 2 +- .../ddagent/TraceMapperV04PayloadTest.java | 439 ++-- .../ddagent/TraceMapperV05PayloadTest.java | 287 ++- .../ddagent/TraceMapperV1PayloadTest.java | 1941 +++++------------ 7 files changed, 921 insertions(+), 1801 deletions(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java index b759ea525f2..84556c1cf9b 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java @@ -10,6 +10,8 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -25,6 +27,7 @@ import datadog.trace.api.Config; import datadog.trace.api.ProcessTags; import datadog.trace.api.ProtocolVersion; +import datadog.trace.api.config.OtlpConfig; import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags; import datadog.trace.common.sampling.RateByServiceTraceSampler; import datadog.trace.common.writer.RemoteApi.Response; @@ -37,7 +40,6 @@ import datadog.trace.core.DDSpan; import datadog.trace.core.DDSpanContext; import datadog.trace.core.propagation.PropagationTags; -import datadog.trace.junit.utils.tabletest.TableTestTypeConverters; import java.io.IOException; import java.lang.reflect.Field; import java.nio.ByteBuffer; @@ -53,6 +55,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import datadog.trace.test.junit.utils.tabletest.TableTestTypeConverters; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import org.junit.jupiter.api.Assertions; @@ -566,6 +569,50 @@ void testMetaStructSupportOnTheEncodedSpans() throws IOException { } } + @TableTest({ + "scenario | otlpSpanMetrics | nativeMetrics | expectedComputesStats", + "gap case: OTLP span metrics on, native off | true | false | true ", + "neither pipeline computes stats | false | false | false ", + "native stats on (regression guard) | false | true | true ", + "both on | true | true | true " + }) + void testDatadogClientComputedStatsHeaderSetWhenEitherStatsPipelineIsEnabled( + boolean otlpSpanMetrics, boolean nativeMetrics, boolean expectedComputesStats) { + datadog.trace.test.junit.utils.config.WithConfigExtension.injectSysConfig( + OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED, String.valueOf(otlpSpanMetrics)); + + JavaTestHttpServer agent = + JavaTestHttpServer.httpServer( + s -> + s.handlers(h -> h.put("v0.4/traces", api -> api.getResponse().status(200).send()))); + try { + HttpUrl agentUrl = HttpUrl.get(agent.getAddress().toString()); + + // Mock feature discovery so the native-stats pipeline signal can be controlled independently + // of what the (embedded) agent advertises. supportsMetrics() reflects agent-side client + // stats support. + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + when(discovery.getTraceEndpoint()).thenReturn("v0.4/traces"); + when(discovery.supportsMetrics()).thenReturn(nativeMetrics); + when(discovery.state()).thenReturn(null); + + OkHttpClient client = OkHttpUtils.buildHttpClient(agentUrl, 1000); + // nativeMetrics is the constructor flag WriterFactory sets from + // Config.isTracerMetricsEnabled(). + DDAgentApi api = new DDAgentApi(client, agentUrl, discovery, monitoring, nativeMetrics); + Payload payload = prepareTraces("v0.4/traces", emptyList()); + + Response clientResponse = api.sendSerializedTraces(payload); + + assertTrue(clientResponse.success()); + assertEquals( + expectedComputesStats, + "true".equals(agent.getLastRequest().getHeaders().get("Datadog-Client-Computed-Stats"))); + } finally { + agent.close(); + } + } + // --- Inner types --- static class AgentApiPair { diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterCombinedTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterCombinedTest.java index 284d40b1736..ae9741fe827 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterCombinedTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentWriterCombinedTest.java @@ -36,7 +36,7 @@ import datadog.trace.core.DDSpan; import datadog.trace.core.monitor.HealthMetrics; import datadog.trace.core.monitor.TracerHealthMetrics; -import datadog.trace.junit.utils.config.WithConfig; +import datadog.trace.test.junit.utils.config.WithConfig; import datadog.trace.test.util.Flaky; import java.io.IOException; import java.util.Collections; diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/PrioritizationTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/PrioritizationTest.java index 4bf08ab36d6..7a9a52cd144 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/PrioritizationTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/PrioritizationTest.java @@ -17,7 +17,7 @@ import datadog.trace.common.writer.ddagent.PrioritizationStrategy; import datadog.trace.common.writer.ddagent.PrioritizationStrategy.PublishResult; import datadog.trace.core.DDSpan; -import datadog.trace.junit.utils.converter.PrioritySamplingConverter; +import datadog.trace.test.junit.utils.converter.PrioritySamplingConverter; import datadog.trace.test.util.DDJavaSpecification; import java.util.Collections; import java.util.List; diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java index 63fe347739c..39b41fdaa44 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java @@ -19,7 +19,7 @@ import datadog.trace.core.DDSpanContext; import datadog.trace.core.PendingTrace; import datadog.trace.core.monitor.HealthMetrics; -import datadog.trace.junit.utils.converter.PrioritySamplingConverter; +import datadog.trace.test.junit.utils.converter.PrioritySamplingConverter; import datadog.trace.test.util.DDJavaSpecification; import java.lang.reflect.Method; import java.util.ArrayList; diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java index aa290d652ea..15483b94c95 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV04PayloadTest.java @@ -2,10 +2,13 @@ import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.DD_MEASURED; import static datadog.trace.common.writer.TraceGenerator.generateRandomTraces; +import static datadog.trace.common.writer.ddagent.PayloadVerifiers.assertEqualsWithNullAsEmpty; +import static datadog.trace.common.writer.ddagent.PayloadVerifiers.unpackNumber; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.params.provider.Arguments.arguments; import datadog.communication.serialization.ByteBufferConsumer; @@ -18,63 +21,49 @@ import datadog.trace.api.ProcessTags; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.common.writer.Payload; -import datadog.trace.common.writer.TraceGenerator; +import datadog.trace.common.writer.TraceGenerator.PojoSpan; import datadog.trace.core.DDSpanContext; -import datadog.trace.test.util.DDJavaSpecification; +import datadog.trace.test.junit.utils.config.WithConfigExtension; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Stream; -import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.msgpack.core.MessageFormat; import org.msgpack.core.MessagePack; import org.msgpack.core.MessageUnpacker; -import org.tabletest.junit.TableTest; -class TraceMapperV04PayloadTest extends DDJavaSpecification { +@ExtendWith(WithConfigExtension.class) +class TraceMapperV04PayloadTest { - @TableTest({ - "scenario | bufferSize | traceCount | lowCardinality", - "0 traces low card | 20480 | 0 | true ", - "1 trace low card 20k | 20480 | 1 | true ", - "1 trace low card 30k | 30720 | 1 | true ", - "2 traces low card | 30720 | 2 | true ", - "0 traces high card | 20480 | 0 | false ", - "1 trace high card 20k | 20480 | 1 | false ", - "1 trace high card 30k | 30720 | 1 | false ", - "2 traces high card | 30720 | 2 | false ", - "0 traces 100k low | 102400 | 0 | true ", - "1 trace 100k low | 102400 | 1 | true ", - "10 traces 100k low | 102400 | 10 | true ", - "100 traces 100k low | 102400 | 100 | true ", - "1000 traces 100k low | 102400 | 1000 | true ", - "0 traces 100k high | 102400 | 0 | false ", - "1 trace 100k high | 102400 | 1 | false ", - "10 traces 100k high | 102400 | 10 | false ", - "100 traces 100k high | 102400 | 100 | false ", - "1000 traces 100k high | 102400 | 1000 | false " - }) - void testTracesWrittenCorrectly(int bufferSize, int traceCount, boolean lowCardinality) { - List> traces = generateRandomTraces(traceCount, lowCardinality); + // Keep the ProcessTags static in sync with the (per-test rebuilt) Config, the way DDSpecification + // did for the original Spock tests. Runs after WithConfigExtension has rebuilt Config. + @BeforeEach + void syncProcessTags() { + ProcessTags.reset(Config.get()); + } + + @ParameterizedTest(name = "buffer={0} traces={1} lowCardinality={2}") + @MethodSource("tracesWrittenCorrectlyArguments") + void tracesWrittenCorrectly(int bufferSize, int traceCount, boolean lowCardinality) { + List> traces = generateRandomTraces(traceCount, lowCardinality); TraceMapperV0_4 traceMapper = new TraceMapperV0_4(); PayloadVerifier verifier = new PayloadVerifier(traces, traceMapper); MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(bufferSize, verifier)); - boolean tracesFitInBuffer = true; - for (List trace : traces) { + for (List trace : traces) { if (!packer.format(trace, traceMapper)) { verifier.skipLargeTrace(); tracesFitInBuffer = false; - // in the real life the mapper is always reset each trace. + // in the real like the mapper is always reset each trace. // here we need to force it when we fail since the buffer will be reset as well traceMapper.reset(); } @@ -86,18 +75,34 @@ void testTracesWrittenCorrectly(int bufferSize, int traceCount, boolean lowCardi } } - static Stream testFull64BitTraceAndSpanIdentifiersArguments() { + private static Stream tracesWrittenCorrectlyArguments() { return Stream.of( - arguments(DD64bTraceId.ONE, 2L, 3L), - arguments(DD64bTraceId.MAX, 2L, 3L), - arguments(DD64bTraceId.from(-10), -11L, -12L)); + arguments(20 << 10, 0, true), + arguments(20 << 10, 1, true), + arguments(30 << 10, 1, true), + arguments(30 << 10, 2, true), + arguments(20 << 10, 0, false), + arguments(20 << 10, 1, false), + arguments(30 << 10, 1, false), + arguments(30 << 10, 2, false), + arguments(100 << 10, 0, true), + arguments(100 << 10, 1, true), + arguments(100 << 10, 10, true), + arguments(100 << 10, 100, true), + arguments(100 << 10, 1000, true), + arguments(100 << 10, 0, false), + arguments(100 << 10, 1, false), + arguments(100 << 10, 10, false), + arguments(100 << 10, 100, false), + arguments(100 << 10, 1000, false)); } - @ParameterizedTest - @MethodSource("testFull64BitTraceAndSpanIdentifiersArguments") - void testFull64BitTraceAndSpanIdentifiers(DDTraceId traceId, long spanId, long parentId) { - TraceGenerator.PojoSpan span = - new TraceGenerator.PojoSpan( + @ParameterizedTest(name = "{0}") + @MethodSource("fullSixtyFourBitTraceAndSpanIdentifiersArguments") + void fullSixtyFourBitTraceAndSpanIdentifiers( + String scenario, DDTraceId traceId, long spanId, long parentId) { + PojoSpan span = + new PojoSpan( "service", "operation", "resource", @@ -114,8 +119,7 @@ void testFull64BitTraceAndSpanIdentifiers(DDTraceId traceId, long spanId, long p 0, 0, "origin"); - List> traces = - Collections.singletonList(Collections.singletonList(span)); + List> traces = Collections.singletonList(Collections.singletonList(span)); TraceMapperV0_4 traceMapper = new TraceMapperV0_4(); PayloadVerifier verifier = new PayloadVerifier(traces, traceMapper); MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(20 << 10, verifier)); @@ -126,10 +130,17 @@ void testFull64BitTraceAndSpanIdentifiers(DDTraceId traceId, long spanId, long p verifier.verifyTracesConsumed(); } + private static Stream fullSixtyFourBitTraceAndSpanIdentifiersArguments() { + return Stream.of( + arguments("ONE", DD64bTraceId.ONE, 2L, 3L), + arguments("MAX", DD64bTraceId.MAX, 2L, 3L), + arguments("negative", DD64bTraceId.from(-10), -11L, -12L)); + } + @Test - void testMetaStructSupport() { - TraceGenerator.PojoSpan span = - new TraceGenerator.PojoSpan( + void metaStructSupport() { + PojoSpan span = + new PojoSpan( "service", "operation", "resource", @@ -146,34 +157,31 @@ void testMetaStructSupport() { 0, 0, "origin"); - - StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); - List> stackList = new ArrayList<>(); - for (StackTraceElement element : stackTrace) { - Map entry = new HashMap<>(); - entry.put("file", element.getFileName() != null ? element.getFileName() : ""); - entry.put("class_name", element.getClassName()); - entry.put("function", element.getMethodName()); - stackList.add(entry); + List> stack = new ArrayList<>(); + for (StackTraceElement element : Thread.currentThread().getStackTrace()) { + Map frame = new HashMap<>(); + frame.put("file", element.getFileName() != null ? element.getFileName() : ""); + frame.put("class_name", element.getClassName() != null ? element.getClassName() : ""); + frame.put("function", element.getMethodName() != null ? element.getMethodName() : ""); + stack.add(frame); } - span.setMetaStruct("stack", stackList); - - List> traces = - Collections.singletonList(Collections.singletonList(span)); + span.setMetaStruct("stack", stack); + List> traces = Collections.singletonList(Collections.singletonList(span)); TraceMapperV0_4 traceMapper = new TraceMapperV0_4(); PayloadVerifier verifier = new PayloadVerifier( traces, traceMapper, - (expectedObj, received) -> { - List expected = (List) expectedObj; + (expected, received) -> { MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(received); + List expectedStack = (List) expected; int size = unpacker.unpackArrayHeader(); - assertEquals(expected.size(), size); - for (int i = 0; i < size; i++) { - Map stackEntry = (Map) expected.get(i); + assertEquals(expectedStack.size(), size); + for (Object entry : expectedStack) { + @SuppressWarnings("unchecked") + Map stackEntry = (Map) entry; int fields = unpacker.unpackMapHeader(); - for (int j = 0; j < fields; j++) { + for (int f = 0; f < fields; ++f) { String field = unpacker.unpackString(); assertEquals(stackEntry.get(field), unpacker.unpackString()); } @@ -188,13 +196,12 @@ void testMetaStructSupport() { } @Test - void testProcessTagsSerialization() { + void processTagsSerialization() { assertNotNull(ProcessTags.getTagsForSerialization()); - - List spans = new ArrayList<>(); - for (int spanId = 1; spanId <= 2; spanId++) { + List spans = new ArrayList<>(); + for (long spanId = 1; spanId <= 2; ++spanId) { spans.add( - new TraceGenerator.PojoSpan( + new PojoSpan( "service", "operation", "resource", @@ -213,7 +220,7 @@ void testProcessTagsSerialization() { "origin")); } - List> traces = Collections.singletonList(spans); + List> traces = Collections.singletonList(spans); TraceMapperV0_4 traceMapper = new TraceMapperV0_4(); PayloadVerifier verifier = new PayloadVerifier(traces, traceMapper); MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(20 << 10, verifier)); @@ -224,28 +231,24 @@ void testProcessTagsSerialization() { verifier.verifyTracesConsumed(); } - // --- Inner classes --- - - private interface MetaStructVerifier { - void verify(Object expected, byte[] received) throws IOException; - } + private static final class PayloadVerifier implements ByteBufferConsumer { - private static final class PayloadVerifier implements ByteBufferConsumer, WritableByteChannel { - - private final List> expectedTraces; + private final List> expectedTraces; private final TraceMapperV0_4 mapper; - private ByteBuffer captured = ByteBuffer.allocate(200 << 10); - private final MetaStructVerifier metaStructVerifier; + private final MetaStructVerifier metaStructVerifier; + private final PayloadVerifiers.CapturingChannel channel = + new PayloadVerifiers.CapturingChannel(200 << 10); + private int position = 0; - private PayloadVerifier(List> traces, TraceMapperV0_4 mapper) { + private PayloadVerifier(List> traces, TraceMapperV0_4 mapper) { this(traces, mapper, null); } private PayloadVerifier( - List> traces, + List> traces, TraceMapperV0_4 mapper, - MetaStructVerifier metaStructVerifier) { + MetaStructVerifier metaStructVerifier) { this.expectedTraces = traces; this.mapper = mapper; this.metaStructVerifier = metaStructVerifier; @@ -263,187 +266,139 @@ public void accept(int messageCount, ByteBuffer buffer) { int processTagsCount = 0; try { Payload payload = mapper.newPayload().withBody(messageCount, buffer); - payload.writeTo(this); - captured.flip(); - try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(captured)) { - int traceCount = unpacker.unpackArrayHeader(); - for (int i = 0; i < traceCount; ++i) { - List expectedTrace = expectedTraces.get(position++); - int spanCount = unpacker.unpackArrayHeader(); - assertEquals(expectedTrace.size(), spanCount); - for (int k = 0; k < spanCount; ++k) { - TraceGenerator.PojoSpan expectedSpan = expectedTrace.get(k); - int elementCount = unpacker.unpackMapHeader(); - boolean hasMetaStruct = !expectedSpan.getMetaStruct().isEmpty(); - assertEquals(hasMetaStruct ? 13 : 12, elementCount); - assertEquals("service", unpacker.unpackString()); - String serviceName = unpacker.unpackString(); - assertEqualsWithNullAsEmpty(expectedSpan.getServiceName(), serviceName); - assertEquals("name", unpacker.unpackString()); - String operationName = unpacker.unpackString(); - assertEqualsWithNullAsEmpty(expectedSpan.getOperationName(), operationName); - assertEquals("resource", unpacker.unpackString()); - String resourceName = unpacker.unpackString(); - assertEqualsWithNullAsEmpty(expectedSpan.getResourceName(), resourceName); - assertEquals("trace_id", unpacker.unpackString()); - long traceId = unpacker.unpackValue().asNumberValue().toLong(); - assertEquals(expectedSpan.getTraceId().toLong(), traceId); - assertEquals("span_id", unpacker.unpackString()); - long spanId = unpacker.unpackValue().asNumberValue().toLong(); - assertEquals(expectedSpan.getSpanId(), spanId); - assertEquals("parent_id", unpacker.unpackString()); - long parentId = unpacker.unpackValue().asNumberValue().toLong(); - assertEquals(expectedSpan.getParentId(), parentId); - assertEquals("start", unpacker.unpackString()); - long startTime = unpacker.unpackLong(); - assertEquals(expectedSpan.getStartTime(), startTime); - assertEquals("duration", unpacker.unpackString()); - long duration = unpacker.unpackLong(); - assertEquals(expectedSpan.getDurationNano(), duration); - assertEquals("type", unpacker.unpackString()); - String type = unpacker.unpackString(); - assertEquals(expectedSpan.getType(), type); - assertEquals("error", unpacker.unpackString()); - int error = unpacker.unpackInt(); - assertEquals(expectedSpan.getError(), error); - assertEquals("metrics", unpacker.unpackString()); - int metricsSize = unpacker.unpackMapHeader(); - HashMap metrics = new HashMap<>(); - for (int j = 0; j < metricsSize; ++j) { - String key = unpacker.unpackString(); - Number metricValue = null; - MessageFormat format = unpacker.getNextFormat(); - switch (format) { - case NEGFIXINT: - case POSFIXINT: - case INT8: - case UINT8: - case INT16: - case UINT16: - case INT32: - case UINT32: - metricValue = unpacker.unpackInt(); - break; - case INT64: - case UINT64: - metricValue = unpacker.unpackLong(); - break; - case FLOAT32: - metricValue = unpacker.unpackFloat(); - break; - case FLOAT64: - metricValue = unpacker.unpackDouble(); - break; - default: - Assertions.fail("Unexpected type in metrics values: " + format); - } - if (DD_MEASURED.toString().equals(key)) { - assertTrue(metricValue.intValue() == 1 || !expectedSpan.isMeasured()); - } else if (DDSpanContext.PRIORITY_SAMPLING_KEY.equals(key)) { - // check that priority sampling is only on first and last span - if (k == 0 || k == spanCount - 1) { - assertEquals(expectedSpan.samplingPriority(), metricValue.intValue()); - } else { - assertFalse(expectedSpan.hasSamplingPriority()); - } + payload.writeTo(channel); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(channel.flipForReading()); + int traceCount = unpacker.unpackArrayHeader(); + for (int i = 0; i < traceCount; ++i) { + List expectedTrace = expectedTraces.get(position++); + int spanCount = unpacker.unpackArrayHeader(); + assertEquals(expectedTrace.size(), spanCount); + for (int k = 0; k < spanCount; ++k) { + PojoSpan expectedSpan = expectedTrace.get(k); + int elementCount = unpacker.unpackMapHeader(); + boolean hasMetaStruct = !expectedSpan.getMetaStruct().isEmpty(); + assertEquals(hasMetaStruct ? 13 : 12, elementCount); + assertEquals("service", unpacker.unpackString()); + String serviceName = unpacker.unpackString(); + assertEqualsWithNullAsEmpty(expectedSpan.getServiceName(), serviceName); + assertEquals("name", unpacker.unpackString()); + String operationName = unpacker.unpackString(); + assertEqualsWithNullAsEmpty(expectedSpan.getOperationName(), operationName); + assertEquals("resource", unpacker.unpackString()); + String resourceName = unpacker.unpackString(); + assertEqualsWithNullAsEmpty(expectedSpan.getResourceName(), resourceName); + assertEquals("trace_id", unpacker.unpackString()); + long traceId = unpacker.unpackValue().asNumberValue().toLong(); + assertEquals(expectedSpan.getTraceId().toLong(), traceId); + assertEquals("span_id", unpacker.unpackString()); + long spanId = unpacker.unpackValue().asNumberValue().toLong(); + assertEquals(expectedSpan.getSpanId(), spanId); + assertEquals("parent_id", unpacker.unpackString()); + long parentId = unpacker.unpackValue().asNumberValue().toLong(); + assertEquals(expectedSpan.getParentId(), parentId); + assertEquals("start", unpacker.unpackString()); + long startTime = unpacker.unpackLong(); + assertEquals(expectedSpan.getStartTime(), startTime); + assertEquals("duration", unpacker.unpackString()); + long duration = unpacker.unpackLong(); + assertEquals(expectedSpan.getDurationNano(), duration); + assertEquals("type", unpacker.unpackString()); + String type = unpacker.unpackString(); + assertEquals(expectedSpan.getType(), type); + assertEquals("error", unpacker.unpackString()); + int error = unpacker.unpackInt(); + assertEquals(expectedSpan.getError(), error); + assertEquals("metrics", unpacker.unpackString()); + int metricsSize = unpacker.unpackMapHeader(); + HashMap metrics = new HashMap<>(); + for (int j = 0; j < metricsSize; ++j) { + String key = unpacker.unpackString(); + Number n = unpackNumber(unpacker, key); + if (DD_MEASURED.toString().equals(key)) { + assertTrue( + (n.intValue() == 1 && expectedSpan.isMeasured()) || !expectedSpan.isMeasured()); + } else if (DDSpanContext.PRIORITY_SAMPLING_KEY.equals(key)) { + // check that priority sampling is only on first and last span + if (k == 0 || k == spanCount - 1) { + assertEquals(expectedSpan.samplingPriority(), n.intValue()); } else { - metrics.put(key, metricValue); + assertFalse(expectedSpan.hasSamplingPriority()); } + } else { + metrics.put(key, n); } - metrics.forEach( - (key, value) -> { - if (value instanceof Double || value instanceof Float) { - assertEquals( - ((Number) expectedSpan.getTag(key)).doubleValue(), - value.doubleValue(), - 0.001); - } else { - // Groovy compared numerically, Java requires explicit long comparison to - // avoid - // Long/Integer type mismatch from different msgpack integer encoding widths - assertEquals( - ((Number) expectedSpan.getTag(key)).longValue(), value.longValue()); - } - }); - assertEquals("meta", unpacker.unpackString()); - int metaSize = unpacker.unpackMapHeader(); - HashMap meta = new HashMap<>(); - for (int j = 0; j < metaSize; ++j) { - meta.put(unpacker.unpackString(), unpacker.unpackString()); + } + for (Map.Entry metric : metrics.entrySet()) { + if (metric.getValue() instanceof Double || metric.getValue() instanceof Float) { + assertEquals( + ((Number) expectedSpan.getTag(metric.getKey())).doubleValue(), + metric.getValue().doubleValue(), + 0.001); + } else { + // Integer-typed metrics round-trip through msgpack's minimal encoding, so a Long + // tag can come back as an Integer (and vice versa). Compare numerically. + assertEquals( + ((Number) expectedSpan.getTag(metric.getKey())).longValue(), + metric.getValue().longValue()); } - for (Map.Entry entry : meta.entrySet()) { - if (Tags.HTTP_STATUS.equals(entry.getKey())) { - assertEquals(String.valueOf(expectedSpan.getHttpStatusCode()), entry.getValue()); - } else if (DDTags.ORIGIN_KEY.equals(entry.getKey())) { - assertEquals(expectedSpan.getOrigin(), entry.getValue()); - } else if (DDTags.PROCESS_TAGS.equals(entry.getKey())) { - assertTrue(Config.get().isExperimentalPropagateProcessTagsEnabled()); - assertEquals(0, k); - assertEquals(ProcessTags.getTagsForSerialization().toString(), entry.getValue()); - processTagsCount++; + } + assertEquals("meta", unpacker.unpackString()); + int metaSize = unpacker.unpackMapHeader(); + HashMap meta = new HashMap<>(); + for (int j = 0; j < metaSize; ++j) { + meta.put(unpacker.unpackString(), unpacker.unpackString()); + } + for (Map.Entry entry : meta.entrySet()) { + if (Tags.HTTP_STATUS.equals(entry.getKey())) { + assertEquals(String.valueOf(expectedSpan.getHttpStatusCode()), entry.getValue()); + } else if (DDTags.ORIGIN_KEY.equals(entry.getKey())) { + assertEquals(expectedSpan.getOrigin(), entry.getValue()); + } else if (DDTags.PROCESS_TAGS.equals(entry.getKey())) { + assertTrue(Config.get().isExperimentalPropagateProcessTagsEnabled()); + assertEquals(0, k); + assertEquals(ProcessTags.getTagsForSerialization().toString(), entry.getValue()); + processTagsCount++; + } else { + Object tag = expectedSpan.getTag(entry.getKey()); + if (tag != null) { + assertEquals(String.valueOf(tag), entry.getValue()); } else { - Object tag = expectedSpan.getTag(entry.getKey()); - if (null != tag) { - assertEquals(String.valueOf(tag), entry.getValue()); - } else { - assertEquals(expectedSpan.getBaggage().get(entry.getKey()), entry.getValue()); - } + assertEquals(expectedSpan.getBaggage().get(entry.getKey()), entry.getValue()); } } - if (hasMetaStruct) { - Map metaStruct = expectedSpan.getMetaStruct(); - assertEquals("meta_struct", unpacker.unpackString()); - int metaStructSize = unpacker.unpackMapHeader(); - for (int j = 0; j < metaStructSize; ++j) { - String field = unpacker.unpackString(); - if (metaStructVerifier != null) { - byte[] binary = new byte[unpacker.unpackBinaryHeader()]; - unpacker.readPayload(binary); - metaStructVerifier.verify(metaStruct.get(field), binary); - } + } + if (hasMetaStruct) { + Map metaStruct = expectedSpan.getMetaStruct(); + assertEquals("meta_struct", unpacker.unpackString()); + int metaStructSize = unpacker.unpackMapHeader(); + for (int j = 0; j < metaStructSize; ++j) { + String field = unpacker.unpackString(); + if (metaStructVerifier != null) { + byte[] binary = new byte[unpacker.unpackBinaryHeader()]; + unpacker.readPayload(binary); + metaStructVerifier.verify(metaStruct.get(field), binary); } } } } } } catch (IOException e) { - Assertions.fail(e.getMessage()); + fail(e.getMessage()); } finally { mapper.reset(); - captured.position(0); - captured.limit(captured.capacity()); + channel.resetForWriting(); assertEquals( Config.get().isExperimentalPropagateProcessTagsEnabled() ? 1 : 0, processTagsCount); } } - @Override - public int write(ByteBuffer src) { - if (captured.remaining() < src.remaining()) { - ByteBuffer newBuffer = ByteBuffer.allocate(captured.capacity() + src.capacity()); - captured.flip(); - newBuffer.put(captured); - captured = newBuffer; - return write(src); - } - captured.put(src); - return src.position(); - } - void verifyTracesConsumed() { assertEquals(expectedTraces.size(), position); } - - @Override - public boolean isOpen() { - return true; - } - - @Override - public void close() {} } - private static void assertEqualsWithNullAsEmpty(CharSequence expected, CharSequence actual) { - assertEquals(expected == null ? "" : expected.toString(), actual.toString()); + @FunctionalInterface + private interface MetaStructVerifier { + void verify(E expected, byte[] received) throws IOException; } } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV05PayloadTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV05PayloadTest.java index b35ea8fa8c3..6dedd2e9ef6 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV05PayloadTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV05PayloadTest.java @@ -3,10 +3,14 @@ import static datadog.trace.api.config.GeneralConfig.EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.DD_MEASURED; import static datadog.trace.common.writer.TraceGenerator.generateRandomTraces; +import static datadog.trace.common.writer.ddagent.PayloadVerifiers.assertEqualsWithNullAsEmpty; +import static datadog.trace.common.writer.ddagent.PayloadVerifiers.unpackNumber; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.params.provider.Arguments.arguments; import datadog.communication.serialization.ByteBufferConsumer; import datadog.communication.serialization.FlushingBuffer; @@ -19,13 +23,12 @@ import datadog.trace.api.sampling.PrioritySampling; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.common.writer.Payload; -import datadog.trace.common.writer.TraceGenerator; +import datadog.trace.common.writer.TraceGenerator.PojoSpan; import datadog.trace.core.DDSpanContext; -import datadog.trace.junit.utils.config.WithConfig; -import datadog.trace.test.util.DDJavaSpecification; +import datadog.trace.test.junit.utils.config.WithConfig; +import datadog.trace.test.junit.utils.config.WithConfigExtension; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -33,34 +36,49 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; -import org.junit.jupiter.api.Assertions; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.msgpack.core.MessageFormat; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.msgpack.core.MessagePack; import org.msgpack.core.MessageUnpacker; -import org.tabletest.junit.TableTest; -class TraceMapperV05PayloadTest extends DDJavaSpecification { +@ExtendWith(WithConfigExtension.class) +class TraceMapperV05PayloadTest { + // Keep the ProcessTags static in sync with the (per-test rebuilt) Config, the way DDSpecification + // did for the original Spock tests. Runs after WithConfigExtension has rebuilt Config. @BeforeEach - void resetProcessTags() { - // Sync ProcessTags.enabled with the current Config (which may be modified by @WithConfig) + void syncProcessTags() { ProcessTags.reset(Config.get()); } - @WithConfig(key = EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED, value = "false") + // A method-level @WithConfig below disables process tags, but WithConfigExtension only rebuilds + // Config after each test -- it does not reset the ProcessTags static. Reset it once the class is + // done so a disabled state can't leak into other tests sharing this Gradle JVM. By afterAll the + // extension has already restored Config to its defaults. + @AfterAll + static void resetProcessTags() { + ProcessTags.reset(Config.get()); + } + + // disable process tags since they are only on the first span of the chunk otherwise the + // calculation woes @Test - void testBodyOverflowCausesFlush() { - // disable process tags since they are only on the first span of the chunk otherwise the - // calculation woes 4x 36 ASCII characters and 2 bytes of msgpack string prefix + @WithConfig(key = EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED, value = "false") + void bodyOverflowCausesAFlush() { + // 4x 36 ASCII characters and 2 bytes of msgpack string prefix int dictionarySpacePerTrace = 4 * (36 + 2); // enough space for two traces with distinct string values, plus the header int dictionarySize = dictionarySpacePerTrace * 2 + 5; TraceMapperV0_5 traceMapper = new TraceMapperV0_5(dictionarySize); - List repeatedTrace = + List repeatedTrace = Collections.singletonList( - new TraceGenerator.PojoSpan( + new PojoSpan( UUID.randomUUID().toString(), UUID.randomUUID().toString(), UUID.randomUUID().toString(), @@ -81,86 +99,83 @@ void testBodyOverflowCausesFlush() { // 30KB body int bufferSize = 30 << 10; int tracesRequiredToOverflowBody = (int) Math.ceil((double) bufferSize / traceSize) + 1; - List> traces = new ArrayList<>(tracesRequiredToOverflowBody); + List> traces = new ArrayList<>(tracesRequiredToOverflowBody); for (int i = 0; i < tracesRequiredToOverflowBody; ++i) { traces.add(repeatedTrace); } // the last one won't be flushed - List> flushedTraces = new ArrayList<>(traces); + List> flushedTraces = new ArrayList<>(traces); flushedTraces.remove(traces.size() - 1); // need space for the overflowing buffer, the dictionary, and two small array headers PayloadVerifier verifier = new PayloadVerifier(flushedTraces, traceMapper, bufferSize + dictionarySize + 1 + 1 + 5); MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(bufferSize, verifier)); - - for (List trace : traces) { + for (List trace : traces) { packer.format(trace, traceMapper); } - verifier.verifyTracesConsumed(); } - @TableTest({ - "scenario | bufferSize | dictionarySize | traceCount | lowCardinality", - "0 traces 10k buf 10k dict low | 10240 | 10240 | 0 | true ", - "1 trace 10k buf 10k dict low | 10240 | 10240 | 1 | true ", - "10 traces 10k buf 10k dict low | 10240 | 10240 | 10 | true ", - "100 traces 10k buf 10k dict low | 10240 | 10240 | 100 | true ", - "1 trace 10k buf 100k dict low | 10240 | 102400 | 1 | true ", - "10 traces 10k buf 100k dict low | 10240 | 102400 | 10 | true ", - "100 traces 10k buf 100k dict low | 10240 | 102400 | 100 | true ", - "0 traces 10k buf 10k dict high | 10240 | 10240 | 0 | false ", - "1 trace 10k buf 10k dict high | 10240 | 10240 | 1 | false ", - "10 traces 10k buf 10k dict high | 10240 | 10240 | 10 | false ", - "100 traces 10k buf 10k dict high | 10240 | 10240 | 100 | false ", - "1 trace 10k buf 100k dict high | 10240 | 102400 | 1 | false ", - "10 traces 10k buf 100k dict high | 10240 | 102400 | 10 | false ", - "100 traces 10k buf 100k dict high | 10240 | 102400 | 100 | false ", - "0 traces 100k buf 10k dict low | 102400 | 10240 | 0 | true ", - "1 trace 100k buf 10k dict low | 102400 | 10240 | 1 | true ", - "10 traces 100k buf 10k dict low | 102400 | 10240 | 10 | true ", - "100 traces 100k buf 10k dict low | 102400 | 10240 | 100 | true ", - "1 trace 100k buf 100k dict low | 102400 | 102400 | 1 | true ", - "10 traces 100k buf 100k dict low | 102400 | 102400 | 10 | true ", - "100 traces 100k buf 100k dict low | 102400 | 102400 | 100 | true ", - "0 traces 100k buf 10k dict high | 102400 | 10240 | 0 | false ", - "1 trace 100k buf 10k dict high | 102400 | 10240 | 1 | false ", - "10 traces 100k buf 10k dict high | 102400 | 10240 | 10 | false ", - "100 traces 100k buf 10k dict high | 102400 | 10240 | 100 | false ", - "1 trace 100k buf 100k dict high | 102400 | 102400 | 1 | false ", - "10 traces 100k buf 100k dict high | 102400 | 102400 | 10 | false ", - "100 traces 100k buf 100k dict high | 102400 | 102400 | 100 | false ", - "1000 traces 100k buf 100k dict high | 102400 | 102400 | 1000 | false " - }) - void testDictionaryCompressedTracesWrittenCorrectly( + @ParameterizedTest(name = "buffer={0} dict={1} traces={2} lowCardinality={3}") + @MethodSource("dictionaryCompressedTracesWrittenCorrectlyArguments") + void dictionaryCompressedTracesWrittenCorrectly( int bufferSize, int dictionarySize, int traceCount, boolean lowCardinality) { - List> traces = generateRandomTraces(traceCount, lowCardinality); + List> traces = generateRandomTraces(traceCount, lowCardinality); TraceMapperV0_5 traceMapper = new TraceMapperV0_5(dictionarySize); PayloadVerifier verifier = new PayloadVerifier(traces, traceMapper); MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(bufferSize, verifier)); - boolean tracesFitInBuffer = true; - for (List trace : traces) { + for (List trace : traces) { if (!packer.format(trace, traceMapper)) { verifier.skipLargeTrace(); tracesFitInBuffer = false; } } packer.flush(); - if (tracesFitInBuffer) { verifier.verifyTracesConsumed(); } } + private static Stream dictionaryCompressedTracesWrittenCorrectlyArguments() { + return Stream.of( + arguments(10 << 10, 10 << 10, 0, true), + arguments(10 << 10, 10 << 10, 1, true), + arguments(10 << 10, 10 << 10, 10, true), + arguments(10 << 10, 10 << 10, 100, true), + arguments(10 << 10, 100 << 10, 1, true), + arguments(10 << 10, 100 << 10, 10, true), + arguments(10 << 10, 100 << 10, 100, true), + arguments(10 << 10, 10 << 10, 0, false), + arguments(10 << 10, 10 << 10, 1, false), + arguments(10 << 10, 10 << 10, 10, false), + arguments(10 << 10, 10 << 10, 100, false), + arguments(10 << 10, 100 << 10, 1, false), + arguments(10 << 10, 100 << 10, 10, false), + arguments(10 << 10, 100 << 10, 100, false), + arguments(100 << 10, 10 << 10, 0, true), + arguments(100 << 10, 10 << 10, 1, true), + arguments(100 << 10, 10 << 10, 10, true), + arguments(100 << 10, 10 << 10, 100, true), + arguments(100 << 10, 100 << 10, 1, true), + arguments(100 << 10, 100 << 10, 10, true), + arguments(100 << 10, 100 << 10, 100, true), + arguments(100 << 10, 10 << 10, 0, false), + arguments(100 << 10, 10 << 10, 1, false), + arguments(100 << 10, 10 << 10, 10, false), + arguments(100 << 10, 10 << 10, 100, false), + arguments(100 << 10, 100 << 10, 1, false), + arguments(100 << 10, 100 << 10, 10, false), + arguments(100 << 10, 100 << 10, 1000, false)); + } + @Test - void testProcessTagsSerialization() { + void processTagsSerialization() { assertNotNull(ProcessTags.getTagsForSerialization()); - - List spans = new ArrayList<>(); - for (int spanId = 1; spanId <= 2; spanId++) { + List spans = new ArrayList<>(); + for (long spanId = 1; spanId <= 2; ++spanId) { spans.add( - new TraceGenerator.PojoSpan( + new PojoSpan( "service", "operation", "resource", @@ -179,7 +194,7 @@ void testProcessTagsSerialization() { "origin")); } - List> traces = Collections.singletonList(spans); + List> traces = Collections.singletonList(spans); TraceMapperV0_5 traceMapper = new TraceMapperV0_5(); PayloadVerifier verifier = new PayloadVerifier(traces, traceMapper); MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(20 << 10, verifier)); @@ -190,24 +205,22 @@ void testProcessTagsSerialization() { verifier.verifyTracesConsumed(); } - // --- Inner classes --- + private static final class PayloadVerifier implements ByteBufferConsumer { - private static final class PayloadVerifier implements ByteBufferConsumer, WritableByteChannel { - - private final List> expectedTraces; + private final List> expectedTraces; private final TraceMapperV0_5 mapper; - private ByteBuffer captured; + private final PayloadVerifiers.CapturingChannel channel; + private int position = 0; - private PayloadVerifier(List> traces, TraceMapperV0_5 mapper) { + private PayloadVerifier(List> traces, TraceMapperV0_5 mapper) { this(traces, mapper, 200 << 10); } - private PayloadVerifier( - List> traces, TraceMapperV0_5 mapper, int size) { + private PayloadVerifier(List> traces, TraceMapperV0_5 mapper, int size) { this.expectedTraces = traces; this.mapper = mapper; - this.captured = ByteBuffer.allocate(size); + this.channel = new PayloadVerifiers.CapturingChannel(size); } void skipLargeTrace() { @@ -219,9 +232,8 @@ public void accept(int messageCount, ByteBuffer buffer) { int processTagsCount = 0; try { Payload payload = mapper.newPayload().withBody(messageCount, buffer); - payload.writeTo(this); - captured.flip(); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(captured); + payload.writeTo(channel); + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(channel.flipForReading()); int header = unpacker.unpackArrayHeader(); assertEquals(2, header); int dictionarySize = unpacker.unpackArrayHeader(); @@ -231,11 +243,11 @@ public void accept(int messageCount, ByteBuffer buffer) { } int traceCount = unpacker.unpackArrayHeader(); for (int i = 0; i < traceCount; ++i) { - List expectedTrace = expectedTraces.get(position++); + List expectedTrace = expectedTraces.get(position++); int spanCount = unpacker.unpackArrayHeader(); assertEquals(expectedTrace.size(), spanCount); for (int k = 0; k < spanCount; ++k) { - TraceGenerator.PojoSpan expectedSpan = expectedTrace.get(k); + PojoSpan expectedSpan = expectedTrace.get(k); int elementCount = unpacker.unpackArrayHeader(); assertEquals(12, elementCount); String serviceName = dictionary[unpacker.unpackInt()]; @@ -262,71 +274,41 @@ public void accept(int messageCount, ByteBuffer buffer) { meta.put(dictionary[unpacker.unpackInt()], dictionary[unpacker.unpackInt()]); } for (Map.Entry entry : meta.entrySet()) { - switch (entry.getKey()) { - case Tags.HTTP_STATUS: - assertEquals(String.valueOf(expectedSpan.getHttpStatusCode()), entry.getValue()); - break; - case DDTags.ORIGIN_KEY: - assertEquals(expectedSpan.getOrigin(), entry.getValue()); - break; - case DDTags.PROCESS_TAGS: - processTagsCount++; - assertTrue(Config.get().isExperimentalPropagateProcessTagsEnabled()); - assertEquals(0, k); - assertEquals(ProcessTags.getTagsForSerialization().toString(), entry.getValue()); - break; - default: - Object tag = expectedSpan.getTag(entry.getKey()); - if (null != tag) { - assertEquals(String.valueOf(tag), entry.getValue()); - } else { - assertEquals(expectedSpan.getBaggage().get(entry.getKey()), entry.getValue()); - } - break; + if (Tags.HTTP_STATUS.equals(entry.getKey())) { + assertEquals(String.valueOf(expectedSpan.getHttpStatusCode()), entry.getValue()); + } else if (DDTags.ORIGIN_KEY.equals(entry.getKey())) { + assertEquals(expectedSpan.getOrigin(), entry.getValue()); + } else if (DDTags.PROCESS_TAGS.equals(entry.getKey())) { + processTagsCount++; + assertTrue(Config.get().isExperimentalPropagateProcessTagsEnabled()); + assertEquals(0, k); + assertEquals(ProcessTags.getTagsForSerialization().toString(), entry.getValue()); + } else { + Object tag = expectedSpan.getTag(entry.getKey()); + if (tag != null) { + assertEquals(String.valueOf(tag), entry.getValue()); + } else { + assertEquals(expectedSpan.getBaggage().get(entry.getKey()), entry.getValue()); + } } } int metricsSize = unpacker.unpackMapHeader(); HashMap metrics = new HashMap<>(); for (int j = 0; j < metricsSize; ++j) { String key = dictionary[unpacker.unpackInt()]; - Number metricValue = null; - MessageFormat format = unpacker.getNextFormat(); - switch (format) { - case NEGFIXINT: - case POSFIXINT: - case INT8: - case UINT8: - case INT16: - case UINT16: - case INT32: - case UINT32: - metricValue = unpacker.unpackInt(); - break; - case INT64: - case UINT64: - metricValue = unpacker.unpackLong(); - break; - case FLOAT32: - metricValue = unpacker.unpackFloat(); - break; - case FLOAT64: - metricValue = unpacker.unpackDouble(); - break; - default: - Assertions.fail( - "Unexpected type in metrics values: " + format + " for key " + key); - } + Number n = unpackNumber(unpacker, key); if (DD_MEASURED.toString().equals(key)) { - assertTrue(metricValue.intValue() == 1 || !expectedSpan.isMeasured()); + assertTrue( + (n.intValue() == 1 && expectedSpan.isMeasured()) || !expectedSpan.isMeasured()); } else if (DDSpanContext.PRIORITY_SAMPLING_KEY.equals(key)) { // check that priority sampling is only on first and last span if (k == 0 || k == spanCount - 1) { - assertEquals(expectedSpan.samplingPriority(), metricValue.intValue()); + assertEquals(expectedSpan.samplingPriority(), n.intValue()); } else { assertFalse(expectedSpan.hasSamplingPriority()); } } else { - metrics.put(key, metricValue); + metrics.put(key, n); } } for (Map.Entry metric : metrics.entrySet()) { @@ -337,7 +319,8 @@ public void accept(int messageCount, ByteBuffer buffer) { 0.001, metric.getKey()); } else { - // Groovy compared numerically; use longValue() to avoid Long/Integer type mismatch + // Integer-typed metrics round-trip through msgpack's minimal encoding, so a Long + // tag can come back as an Integer (and vice versa). Compare numerically. assertEquals( ((Number) expectedSpan.getTag(metric.getKey())).longValue(), metric.getValue().longValue(), @@ -349,56 +332,32 @@ public void accept(int messageCount, ByteBuffer buffer) { } } } catch (IOException e) { - Assertions.fail(e.getMessage()); + fail(e.getMessage()); } finally { assertEquals( Config.get().isExperimentalPropagateProcessTagsEnabled() ? 1 : 0, processTagsCount); mapper.reset(); - captured.position(0); - captured.limit(captured.capacity()); - } - } - - @Override - public int write(ByteBuffer src) { - if (captured.remaining() < src.remaining()) { - ByteBuffer newBuffer = ByteBuffer.allocate(captured.capacity() + src.remaining()); - captured.flip(); - newBuffer.put(captured); - captured = newBuffer; - return write(src); + channel.resetForWriting(); } - captured.put(src); - return src.position(); } void verifyTracesConsumed() { assertEquals(expectedTraces.size(), position); } - - @Override - public boolean isOpen() { - return true; - } - - @Override - public void close() {} - } - - private static void assertEqualsWithNullAsEmpty(CharSequence expected, CharSequence actual) { - if (null == expected) { - assertEquals("", actual); - } else { - assertEquals(expected.toString(), actual.toString()); - } } - static int calculateSize(List trace) { + private static int calculateSize(List trace) { AtomicInteger size = new AtomicInteger(); MsgPackWriter packer = new MsgPackWriter( new FlushingBuffer( - 1024, (messageCount, buffer) -> size.set(buffer.limit() - buffer.position()))); + 1024, + new ByteBufferConsumer() { + @Override + public void accept(int messageCount, ByteBuffer buffer) { + size.set(buffer.limit() - buffer.position()); + } + })); packer.format(trace, new TraceMapperV0_5(1024)); packer.flush(); return size.get(); diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java index 2f4fd225124..1eacd903b96 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV1PayloadTest.java @@ -1,76 +1,151 @@ package datadog.trace.common.writer.ddagent; +import static datadog.trace.api.DDTags.PROCESS_TAGS; +import static datadog.trace.api.DDTags.SPAN_EVENTS; +import static datadog.trace.api.DDTags.THREAD_ID; +import static datadog.trace.api.DDTags.THREAD_NAME; +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; +import static datadog.trace.bootstrap.instrumentation.api.Tags.COMPONENT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.ENV; +import static datadog.trace.bootstrap.instrumentation.api.Tags.HTTP_STATUS; +import static datadog.trace.bootstrap.instrumentation.api.Tags.HTTP_URL; +import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND; +import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND_CLIENT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.VERSION; import static datadog.trace.common.writer.TraceGenerator.generateRandomTraces; +import static datadog.trace.common.writer.ddagent.PayloadVerifiers.assertEqualsWithNullAsEmpty; +import static datadog.trace.common.writer.ddagent.V1PayloadReader.newStringTable; +import static datadog.trace.common.writer.ddagent.V1PayloadReader.readAttributes; +import static datadog.trace.common.writer.ddagent.V1PayloadReader.readBinary; +import static datadog.trace.common.writer.ddagent.V1PayloadReader.readFirstChunk; +import static datadog.trace.common.writer.ddagent.V1PayloadReader.readFirstSpan; +import static datadog.trace.common.writer.ddagent.V1PayloadReader.readStreamingString; +import static datadog.trace.common.writer.ddagent.V1PayloadReader.skipChunkField; +import static datadog.trace.common.writer.ddagent.V1PayloadReader.skipPayloadField; +import static datadog.trace.common.writer.ddagent.V1PayloadReader.skipSpanField; +import static datadog.trace.common.writer.ddagent.V1PayloadReader.traceIdBytes; +import static datadog.trace.common.writer.ddagent.V1PayloadReader.unpackUnsignedLong; +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.msgpack.core.MessageFormat.FIXSTR; -import static org.msgpack.core.MessageFormat.STR16; -import static org.msgpack.core.MessageFormat.STR32; -import static org.msgpack.core.MessageFormat.STR8; +import static org.junit.jupiter.api.Assertions.fail; import datadog.communication.serialization.ByteBufferConsumer; import datadog.communication.serialization.FlushingBuffer; import datadog.communication.serialization.msgpack.MsgPackWriter; +import datadog.trace.api.Config; import datadog.trace.api.DDSpanId; -import datadog.trace.api.DDTags; import datadog.trace.api.DDTraceId; import datadog.trace.api.ProcessTags; -import datadog.trace.api.sampling.PrioritySampling; import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink; import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags; import datadog.trace.bootstrap.instrumentation.api.SpanAttributes; import datadog.trace.bootstrap.instrumentation.api.SpanLink; -import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.common.writer.Payload; -import datadog.trace.common.writer.TraceGenerator; +import datadog.trace.common.writer.TraceGenerator.PojoSpan; +import datadog.trace.common.writer.ddagent.V1PayloadReader.ChunkField; +import datadog.trace.common.writer.ddagent.V1PayloadReader.PayloadField; +import datadog.trace.common.writer.ddagent.V1PayloadReader.SpanField; +import datadog.trace.common.writer.ddagent.V1PayloadReader.V1SpanEvent; +import datadog.trace.common.writer.ddagent.V1PayloadReader.V1SpanLink; import datadog.trace.core.MetadataConsumer; -import datadog.trace.junit.utils.converter.SamplingMechanismConverter; -import datadog.trace.test.util.DDJavaSpecification; +import datadog.trace.test.junit.utils.config.WithConfigExtension; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; +import java.nio.channels.Channels; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.converter.ConvertWith; +import org.junit.jupiter.api.extension.ExtendWith; import org.msgpack.core.MessageFormat; import org.msgpack.core.MessagePack; import org.msgpack.core.MessageUnpacker; +import org.msgpack.value.ValueType; import org.tabletest.junit.TableTest; -class TraceMapperV1PayloadTest extends DDJavaSpecification { +@ExtendWith(WithConfigExtension.class) +class TraceMapperV1PayloadTest { + + private static final String DECISION_MAKER_TAG = "_dd.p.dm"; + + /** Every field the v1 payload header carries, mirroring {@code TraceMapperV1.buildHeader}. */ + private static final Set EXPECTED_PAYLOAD_FIELD_IDS = + new HashSet<>( + asList( + PayloadField.CONTAINER_ID, + PayloadField.LANGUAGE_NAME, + PayloadField.LANGUAGE_VERSION, + PayloadField.TRACER_VERSION, + PayloadField.RUNTIME_ID, + PayloadField.ENV, + PayloadField.HOSTNAME, + PayloadField.APP_VERSION, + PayloadField.ATTRIBUTES, + PayloadField.CHUNKS)); + + /** Every field a v1 span carries, mirroring {@code TraceMapperV1.encodeSpans}. */ + private static final Set EXPECTED_SPAN_FIELD_IDS = + new HashSet<>( + asList( + SpanField.SERVICE, + SpanField.NAME, + SpanField.RESOURCE, + SpanField.SPAN_ID, + SpanField.PARENT_ID, + SpanField.START, + SpanField.DURATION, + SpanField.ERROR, + SpanField.ATTRIBUTES, + SpanField.TYPE, + SpanField.LINKS, + SpanField.EVENTS, + SpanField.ENV, + SpanField.VERSION, + SpanField.COMPONENT, + SpanField.KIND)); + + // Keep the ProcessTags static in sync with the (per-test rebuilt) Config, the way DDSpecification + // did for the original Spock tests. Runs after WithConfigExtension has rebuilt Config. + @BeforeEach + void syncProcessTags() { + ProcessTags.reset(Config.get()); + } @TableTest({ - "scenario | bufferSize | traceCount | lowCardinality", - "0 traces low card | 20480 | 0 | true ", - "1 trace low card | 20480 | 1 | true ", - "2 traces low card | 30720 | 2 | true ", - "0 traces high card | 20480 | 0 | false ", - "1 trace high card | 20480 | 1 | false ", - "2 traces high card | 30720 | 2 | false ", - "10 traces low card | 102400 | 10 | true ", - "100 traces high card | 102400 | 100 | false " + "scenario | bufferSizeKb | traceCount | lowCardinality", + "no traces, low cardinality | 20 | 0 | true ", + "one trace, low cardinality | 20 | 1 | true ", + "two traces, low cardinality | 30 | 2 | true ", + "no traces, high cardinality | 20 | 0 | false ", + "one trace, high cardinality | 20 | 1 | false ", + "two traces, high cardinality | 30 | 2 | false ", + "ten traces, low cardinality | 100 | 10 | true ", + "hundred traces, high cardinality | 100 | 100 | false " }) - void testTracesWrittenCorrectly(int bufferSize, int traceCount, boolean lowCardinality) { - List> traces = generateRandomTraces(traceCount, lowCardinality); + void tracesWrittenCorrectly( + String scenario, int bufferSizeKb, int traceCount, boolean lowCardinality) { + List> traces = generateRandomTraces(traceCount, lowCardinality); TraceMapperV1 traceMapper = new TraceMapperV1(); PayloadVerifier verifier = new PayloadVerifier(traces, traceMapper); - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(bufferSize, verifier)); + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(bufferSizeKb << 10, verifier)); boolean tracesFitInBuffer = true; - for (List trace : traces) { + for (List trace : traces) { if (!packer.format(trace, traceMapper)) { verifier.skipLargeTrace(); tracesFitInBuffer = false; @@ -85,40 +160,37 @@ void testTracesWrittenCorrectly(int bufferSize, int traceCount, boolean lowCardi } @Test - void testEndpointReturnsV10() { + void endpointReturnsV1() { assertEquals("v1.0", new TraceMapperV1().endpoint()); } - @Test - void testSpanKindValueConversion() { - assertEquals(TraceMapperV1.SPAN_KIND_UNSPECIFIED, TraceMapperV1.getSpanKindValue(null)); - assertEquals( - TraceMapperV1.SPAN_KIND_INTERNAL, TraceMapperV1.getSpanKindValue(Tags.SPAN_KIND_INTERNAL)); - assertEquals( - TraceMapperV1.SPAN_KIND_SERVER, TraceMapperV1.getSpanKindValue(Tags.SPAN_KIND_SERVER)); - assertEquals( - TraceMapperV1.SPAN_KIND_CLIENT, TraceMapperV1.getSpanKindValue(Tags.SPAN_KIND_CLIENT)); - assertEquals( - TraceMapperV1.SPAN_KIND_PRODUCER, TraceMapperV1.getSpanKindValue(Tags.SPAN_KIND_PRODUCER)); - assertEquals( - TraceMapperV1.SPAN_KIND_CONSUMER, TraceMapperV1.getSpanKindValue(Tags.SPAN_KIND_CONSUMER)); - assertEquals(TraceMapperV1.SPAN_KIND_INTERNAL, TraceMapperV1.getSpanKindValue("unknown")); + // expectedKind holds the wire values, which must stay in sync with TraceMapperV1.SPAN_KIND_*. + @TableTest({ + "scenario | spanKind | expectedKind", + "no span.kind tag | | 0 ", + "internal span kind | internal | 1 ", + "server span kind | server | 2 ", + "client span kind | client | 3 ", + "producer span kind | producer | 4 ", + "consumer span kind | consumer | 5 ", + "unrecognized span kind | unknown | 1 " + }) + void spanKindValueConversion(String scenario, String spanKind, int expectedKind) { + assertEquals(expectedKind, TraceMapperV1.getSpanKindValue(spanKind)); } @Test - void testPayloadContainsExpectedHeaderAndChunkFields() throws IOException { + void payloadContainsExpectedHeaderAndChunkFields() throws IOException { Map tags = new HashMap<>(); - tags.put(Tags.ENV, "prod"); - tags.put(Tags.VERSION, "1.2.3"); - tags.put(Tags.COMPONENT, "http-client"); - tags.put(Tags.SPAN_KIND, Tags.SPAN_KIND_CLIENT); + tags.put(ENV, "prod"); + tags.put(VERSION, "1.2.3"); + tags.put(COMPONENT, "http-client"); + tags.put(SPAN_KIND, SPAN_KIND_CLIENT); tags.put("attr.string", "value"); tags.put("attr.bool", true); tags.put("attr.number", 12.5d); - tags.put("_dd.p.dm", "-3"); - - TraceGenerator.PojoSpan span = - new TraceGenerator.PojoSpan( + PojoSpan span = + new PojoSpan( "service-a", "operation-a", "resource-a", @@ -128,205 +200,115 @@ void testPayloadContainsExpectedHeaderAndChunkFields() throws IOException { 1000L, 2000L, 1, - Collections.emptyMap(), + singletonMap(DECISION_MAKER_TAG, "-3"), tags, "web", false, - PrioritySampling.SAMPLER_KEEP, + SAMPLER_KEEP, 200, "rum"); - TraceMapperV1 mapper = new TraceMapperV1(); - byte[] encoded = - serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); - List stringTable = new ArrayList<>(); - stringTable.add(""); - - int payloadFieldCount = unpacker.unpackMapHeader(); - Set payloadFieldsSeen = new HashSet<>(); - int chunkCount = -1; - Map payloadAttributes = null; - - for (int i = 0; i < payloadFieldCount; i++) { - int fieldId = unpacker.unpackInt(); - payloadFieldsSeen.add(fieldId); - switch (fieldId) { - case 2: - case 3: - case 4: - case 5: - case 6: - case 7: - case 8: - case 9: - readStreamingString(unpacker, stringTable); - break; - case 10: - payloadAttributes = readAttributes(unpacker, stringTable); - break; - case 11: + byte[] encoded = serializeV1Payload(span); + List stringTable = newStringTable(); + + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded)) { + int payloadFieldCount = unpacker.unpackMapHeader(); + Set payloadFieldsSeen = new HashSet<>(); + int chunkCount = -1; + Map payloadAttributes = null; + + for (int i = 0; i < payloadFieldCount; i++) { + int fieldId = unpacker.unpackInt(); + payloadFieldsSeen.add(fieldId); + if (fieldId == PayloadField.CHUNKS) { chunkCount = unpacker.unpackArrayHeader(); assertEquals(1, chunkCount); - verifyChunk(unpacker, Collections.singletonList(span), stringTable); - break; - default: - Assertions.fail("Unexpected payload field id: " + fieldId); + verifyChunk(unpacker, singletonList(span), stringTable); + } else if (fieldId == PayloadField.ATTRIBUTES) { + payloadAttributes = readAttributes(unpacker, stringTable); + } else { + skipPayloadField(unpacker, fieldId, stringTable); + } } - } - assertEquals(10, payloadFieldCount); - Set expectedFields = new HashSet<>(); - for (int i = 2; i <= 11; i++) { - expectedFields.add(i); - } - assertEquals(expectedFields, payloadFieldsSeen); - assertEquals(1, chunkCount); - assertNotNull(payloadAttributes); - if (ProcessTags.getTagsForSerialization() == null) { - assertEquals(0, payloadAttributes.size()); - } else { - assertEquals(1, payloadAttributes.size()); - assertEquals( - ProcessTags.getTagsForSerialization().toString(), - payloadAttributes.get(DDTags.PROCESS_TAGS)); + assertEquals(EXPECTED_PAYLOAD_FIELD_IDS.size(), payloadFieldCount); + assertEquals(EXPECTED_PAYLOAD_FIELD_IDS, payloadFieldsSeen); + assertEquals(1, chunkCount); + assertNotNull(payloadAttributes); + CharSequence processTags = ProcessTags.getTagsForSerialization(); + if (processTags == null) { + assertEquals(0, payloadAttributes.size()); + } else { + assertEquals(1, payloadAttributes.size()); + assertEquals(processTags.toString(), payloadAttributes.get(PROCESS_TAGS)); + } } } + // expectedSamplingMechanism 0 is SamplingMechanism.DEFAULT. @TableTest({ - "scenario | decisionMakerTag | expectedSamplingMechanism ", - "null tag | | SamplingMechanism.DEFAULT ", - "simple negative | '-3' | SamplingMechanism.LOCAL_USER_RULE", - "compound | '934086a686-7' | 7 ", - "invalid | 'invalid' | SamplingMechanism.DEFAULT " + "scenario | decisionMakerTag | expectedSamplingMechanism", + "no _dd.p.dm tag | | 0 ", + "negative numeric value | '-3' | 3 ", + "hashed prefix before the dash | '934086a686-7' | 7 ", + "unparseable value | invalid | 0 " }) - void testSamplingMechanismNormalizationFromDdPDm( - String decisionMakerTag, - @ConvertWith(SamplingMechanismConverter.class) int expectedSamplingMechanism) - throws IOException { - Map dmTags = - decisionMakerTag == null - ? Collections.emptyMap() - : Collections.singletonMap("_dd.p.dm", decisionMakerTag); - - TraceGenerator.PojoSpan span = - new TraceGenerator.PojoSpan( - "service-a", - "operation-a", - "resource-a", - DDTraceId.ONE, - 321L, - 0L, - 1000L, - 2000L, - 0, - Collections.emptyMap(), - dmTags, - "custom", - false, - PrioritySampling.SAMPLER_KEEP, - 200, - null); - - TraceMapperV1 mapper = new TraceMapperV1(); - byte[] encoded = - serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); - List stringTable = new ArrayList<>(); - stringTable.add(""); - - unpacker.unpackMapHeader(); - int samplingMechanism = -1; - - for (int i = 0; i < 10; i++) { - int payloadFieldId = unpacker.unpackInt(); - if (payloadFieldId == 11) { - int chunkCount = unpacker.unpackArrayHeader(); - assertEquals(1, chunkCount); - int chunkFieldCount = unpacker.unpackMapHeader(); - for (int j = 0; j < chunkFieldCount; j++) { - int chunkFieldId = unpacker.unpackInt(); - if (chunkFieldId == 7) { - samplingMechanism = unpacker.unpackInt(); - } else { - skipChunkField(unpacker, chunkFieldId, stringTable); - } - } - } else { - skipPayloadField(unpacker, payloadFieldId, stringTable); - } + void samplingMechanismNormalizationFromDecisionMaker( + String scenario, String decisionMakerTag, int expectedSamplingMechanism) throws IOException { + Map baggage = new HashMap<>(); + if (decisionMakerTag != null) { + baggage.put(DECISION_MAKER_TAG, decisionMakerTag); } - assertEquals(expectedSamplingMechanism, samplingMechanism); + byte[] encoded = serializeV1Payload(spanWithBaggage(baggage)); + + assertEquals(expectedSamplingMechanism, readFirstChunk(encoded).getSamplingMechanism()); } @Test - void testSpanIdsAreEncodedAsUnsignedValuesInV1Payloads() throws IOException { + void spanIdsAreEncodedAsUnsignedValues() throws IOException { long spanId = Long.MIN_VALUE + 123L; long parentId = Long.MIN_VALUE + 456L; - TraceGenerator.PojoSpan span = - new TraceGenerator.PojoSpan( - "service-a", - "operation-a", - "resource-a", - DDTraceId.ONE, - spanId, - parentId, - 1000L, - 2000L, - 0, - Collections.emptyMap(), - Collections.emptyMap(), - "web", - false, - PrioritySampling.SAMPLER_KEEP, - 200, - null); - TraceMapperV1 mapper = new TraceMapperV1(); - byte[] encoded = - serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); - List stringTable = new ArrayList<>(); - stringTable.add(""); + byte[] encoded = serializeV1Payload(spanWithIds(spanId, parentId)); + List stringTable = newStringTable(); - unpacker.unpackMapHeader(); Long actualSpanId = null; Long actualParentId = null; + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded)) { + int payloadFieldCount = unpacker.unpackMapHeader(); + for (int i = 0; i < payloadFieldCount; i++) { + int payloadFieldId = unpacker.unpackInt(); + if (payloadFieldId != PayloadField.CHUNKS) { + skipPayloadField(unpacker, payloadFieldId, stringTable); + continue; + } + assertEquals(1, unpacker.unpackArrayHeader()); - for (int i = 0; i < 10; i++) { - int payloadFieldId = unpacker.unpackInt(); - if (payloadFieldId == 11) { - int chunkCount = unpacker.unpackArrayHeader(); - assertEquals(1, chunkCount); int chunkFieldCount = unpacker.unpackMapHeader(); for (int j = 0; j < chunkFieldCount; j++) { int chunkFieldId = unpacker.unpackInt(); - if (chunkFieldId == 4) { - int spanCount = unpacker.unpackArrayHeader(); - assertEquals(1, spanCount); - int spanFieldCount = unpacker.unpackMapHeader(); - for (int k = 0; k < spanFieldCount; k++) { - int spanFieldId = unpacker.unpackInt(); - switch (spanFieldId) { - case 4: - assertEquals(MessageFormat.UINT64, unpacker.getNextFormat()); - actualSpanId = DDSpanId.from(unpacker.unpackBigInteger().toString()); - break; - case 5: - assertEquals(MessageFormat.UINT64, unpacker.getNextFormat()); - actualParentId = DDSpanId.from(unpacker.unpackBigInteger().toString()); - break; - default: - skipSpanField(unpacker, spanFieldId, stringTable); - } - } - } else { + if (chunkFieldId != ChunkField.SPANS) { skipChunkField(unpacker, chunkFieldId, stringTable); + continue; + } + assertEquals(1, unpacker.unpackArrayHeader()); + + int spanFieldCount = unpacker.unpackMapHeader(); + for (int k = 0; k < spanFieldCount; k++) { + int spanFieldId = unpacker.unpackInt(); + switch (spanFieldId) { + case SpanField.SPAN_ID: + actualSpanId = unpackUint64(unpacker); + break; + case SpanField.PARENT_ID: + actualParentId = unpackUint64(unpacker); + break; + default: + skipSpanField(unpacker, spanFieldId, stringTable); + } } } - } else { - skipPayloadField(unpacker, payloadFieldId, stringTable); } } @@ -335,394 +317,183 @@ void testSpanIdsAreEncodedAsUnsignedValuesInV1Payloads() throws IOException { } @Test - void testSpanLinksAreEncodedFromStructuredSpanLinks() throws IOException { - Map linkAttrs = new HashMap<>(); - linkAttrs.put("link.kind", "follows_from"); - linkAttrs.put("context_headers", "tracecontext"); - - List spanLinks = - Arrays.asList( - new TestSpanLink( - DDTraceId.fromHex("11223344556677889900aabbccddeeff"), - DDSpanId.fromHex("000000000000002a"), - (byte) 1, - "dd=s:1", - SpanAttributes.fromMap(linkAttrs)), - new TestSpanLink( - DDTraceId.fromHex("00000000000000000000000000000001"), - DDSpanId.fromHex("0000000000000002"), - (byte) 0, - "", - SpanAttributes.EMPTY)); - - TraceGenerator.PojoSpan span = - new TraceGenerator.PojoSpan( - "service-a", - "operation-a", - "resource-a", - DDTraceId.ONE, - 123L, - 0L, - 1000L, - 2000L, - 0, - Collections.emptyMap(), - Collections.emptyMap(), - "web", - false, - PrioritySampling.SAMPLER_KEEP, - 200, - null, - spanLinks); - - TraceMapperV1 mapper = new TraceMapperV1(); - byte[] encoded = - serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); - List stringTable = new ArrayList<>(); - stringTable.add(""); - - List> links = readFirstSpanLinks(unpacker, stringTable); + void spanLinksAreEncodedFromStructuredSpanLinks() throws IOException { + DDTraceId firstLinkTraceId = DDTraceId.fromHex("11223344556677889900aabbccddeeff"); + long firstLinkSpanId = DDSpanId.fromHex("000000000000002a"); + Map firstLinkAttributes = new HashMap<>(); + firstLinkAttributes.put("link.kind", "follows_from"); + firstLinkAttributes.put("context_headers", "tracecontext"); + DDTraceId secondLinkTraceId = DDTraceId.fromHex("00000000000000000000000000000001"); + long secondLinkSpanId = DDSpanId.fromHex("0000000000000002"); + List spanLinks = new ArrayList<>(); + spanLinks.add( + new TestSpanLink( + firstLinkTraceId, + firstLinkSpanId, + (byte) 1, + "dd=s:1", + SpanAttributes.fromMap(firstLinkAttributes))); + spanLinks.add( + new TestSpanLink(secondLinkTraceId, secondLinkSpanId, (byte) 0, "", SpanAttributes.EMPTY)); + + List links = readFirstSpan(serializeV1Payload(spanWithLinks(spanLinks))).getLinks(); assertEquals(2, links.size()); - assertArrayEquals( - traceIdBytes(DDTraceId.fromHex("11223344556677889900aabbccddeeff")), - (byte[]) links.get(0).get("traceId")); - assertEquals(DDSpanId.fromHex("000000000000002a"), links.get(0).get("spanId")); - assertEquals("dd=s:1", links.get(0).get("tracestate")); - assertEquals(1L, links.get(0).get("flags")); - Map expectedLinkAttrs0 = new HashMap<>(); - expectedLinkAttrs0.put("link.kind", "follows_from"); - expectedLinkAttrs0.put("context_headers", "tracecontext"); - assertEquals(expectedLinkAttrs0, links.get(0).get("attributes")); - - assertArrayEquals( - traceIdBytes(DDTraceId.fromHex("00000000000000000000000000000001")), - (byte[]) links.get(1).get("traceId")); - assertEquals(DDSpanId.fromHex("0000000000000002"), links.get(1).get("spanId")); - assertEquals("", links.get(1).get("tracestate")); - assertEquals(0L, links.get(1).get("flags")); - assertEquals(Collections.emptyMap(), links.get(1).get("attributes")); + V1SpanLink firstLink = links.get(0); + assertArrayEquals(traceIdBytes(firstLinkTraceId), firstLink.getTraceId()); + assertEquals(firstLinkSpanId, firstLink.getSpanId()); + assertEquals("dd=s:1", firstLink.getTraceState()); + assertEquals(1L, firstLink.getTraceFlags()); + assertEquals(firstLinkAttributes, firstLink.getAttributes()); + + V1SpanLink secondLink = links.get(1); + assertArrayEquals(traceIdBytes(secondLinkTraceId), secondLink.getTraceId()); + assertEquals(secondLinkSpanId, secondLink.getSpanId()); + assertEquals("", secondLink.getTraceState()); + assertEquals(0L, secondLink.getTraceFlags()); + assertEquals(emptyMap(), secondLink.getAttributes()); } @Test - void testFirstSpanTagsAreProcessedOnce() { - CountingPojoSpan firstSpan = - new CountingPojoSpan( - "service-a", - "operation-a", - "resource-a", - DDTraceId.ONE, - 123L, - 0L, - 1000L, - 2000L, - 0, - Collections.emptyMap(), - Collections.singletonMap(Tags.HTTP_URL, "http://localhost:7777/"), - "web", - false, - PrioritySampling.SAMPLER_KEEP, - 200, - null); + void firstSpanTagsAreProcessedOnce() { + CountingPojoSpan firstSpan = new CountingPojoSpan("operation-a", "resource-a", 123L, 0L); + CountingPojoSpan secondSpan = new CountingPojoSpan("operation-b", "resource-b", 456L, 123L); + List trace = new ArrayList<>(); + trace.add(firstSpan); + trace.add(secondSpan); - CountingPojoSpan secondSpan = - new CountingPojoSpan( - "service-a", - "operation-b", - "resource-b", - DDTraceId.ONE, - 456L, - 123L, - 1000L, - 2000L, - 0, - Collections.emptyMap(), - Collections.singletonMap(Tags.HTTP_URL, "http://localhost:7777/"), - "web", - false, - PrioritySampling.SAMPLER_KEEP, - 200, - null); - - TraceMapperV1 mapper = new TraceMapperV1(); - - serializeMappedPayload(mapper, Collections.singletonList(Arrays.asList(firstSpan, secondSpan))); + serializeV1Payload(trace); assertEquals(1, firstSpan.processTagsAndBaggageCount); assertEquals(1, secondSpan.processTagsAndBaggageCount); } @Test - void testMissingSpanLinksEncodeEmptyLinks() throws IOException { - TraceGenerator.PojoSpan span = - new TraceGenerator.PojoSpan( - "service-a", - "operation-a", - "resource-a", - DDTraceId.ONE, - 123L, - 0L, - 1000L, - 2000L, - 0, - Collections.emptyMap(), - Collections.emptyMap(), - "web", - false, - PrioritySampling.SAMPLER_KEEP, - 200, - null); + void missingSpanLinksEncodeEmptyLinks() throws IOException { + byte[] encoded = serializeV1Payload(span(emptyMap())); - TraceMapperV1 mapper = new TraceMapperV1(); - byte[] encoded = - serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); - List stringTable = new ArrayList<>(); - stringTable.add(""); - - List> links = readFirstSpanLinks(unpacker, stringTable); - - assertTrue(links.isEmpty()); + assertTrue(readFirstSpan(encoded).getLinks().isEmpty()); } @Test - void testSpanEventsAreEncodedFromEventsTag() throws IOException { - Map eventOneAttrs = new HashMap<>(); - eventOneAttrs.put("str", "v"); - eventOneAttrs.put("int", 42L); - eventOneAttrs.put("double", 12.5d); - eventOneAttrs.put("bool", true); - eventOneAttrs.put("arr", Arrays.asList("x", 7L, 2.5d, false)); - - Map eventOne = new HashMap<>(); - eventOne.put("time_unix_nano", 1234567890L); - eventOne.put("name", "event.one"); - eventOne.put("attributes", eventOneAttrs); - - Map eventTwo = new HashMap<>(); - eventTwo.put("time_unix_nano", 1234567891L); - eventTwo.put("name", "event.two"); - - Map tags = - Collections.singletonMap("events", Arrays.asList(eventOne, eventTwo)); - - TraceGenerator.PojoSpan span = - new TraceGenerator.PojoSpan( - "service-a", - "operation-a", - "resource-a", - DDTraceId.ONE, - 123L, - 0L, - 1000L, - 2000L, - 0, - Collections.emptyMap(), - tags, - "web", - false, - PrioritySampling.SAMPLER_KEEP, - 200, - null); - - TraceMapperV1 mapper = new TraceMapperV1(); - byte[] encoded = - serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); - List stringTable = new ArrayList<>(); - stringTable.add(""); - - List> events = readFirstSpanEvents(unpacker, stringTable); + void spanEventsAreEncodedFromEventsTag() throws IOException { + Map firstEventAttributes = new HashMap<>(); + firstEventAttributes.put("str", "v"); + firstEventAttributes.put("int", 42L); + firstEventAttributes.put("double", 12.5d); + firstEventAttributes.put("bool", true); + firstEventAttributes.put("arr", asList("x", 7L, 2.5d, false)); + Map firstEvent = new HashMap<>(); + firstEvent.put("time_unix_nano", 1234567890L); + firstEvent.put("name", "event.one"); + firstEvent.put("attributes", firstEventAttributes); + Map secondEvent = new HashMap<>(); + secondEvent.put("time_unix_nano", 1234567891L); + secondEvent.put("name", "event.two"); + PojoSpan span = span(singletonMap(SPAN_EVENTS, asList(firstEvent, secondEvent))); + + List events = readFirstSpan(serializeV1Payload(span)).getEvents(); assertEquals(2, events.size()); - assertEquals(1234567890L, events.get(0).get("timeUnixNano")); - assertEquals("event.one", events.get(0).get("name")); - Map decodedAttrs0 = (Map) events.get(0).get("attributes"); - assertEquals("v", decodedAttrs0.get("str")); - assertEquals(42L, decodedAttrs0.get("int")); - assertEquals(12.5d, ((Number) decodedAttrs0.get("double")).doubleValue(), 0.000001d); - assertEquals(true, decodedAttrs0.get("bool")); - assertEquals(Arrays.asList("x", 7L, 2.5d, false), decodedAttrs0.get("arr")); - - assertEquals(1234567891L, events.get(1).get("timeUnixNano")); - assertEquals("event.two", events.get(1).get("name")); - assertEquals(Collections.emptyMap(), events.get(1).get("attributes")); + V1SpanEvent firstDecodedEvent = events.get(0); + assertEquals(1234567890L, firstDecodedEvent.getTimeUnixNano()); + assertEquals("event.one", firstDecodedEvent.getName()); + Map firstDecodedAttributes = firstDecodedEvent.getAttributes(); + assertEquals("v", firstDecodedAttributes.get("str")); + assertEquals(42L, firstDecodedAttributes.get("int")); + assertAttributeValueEquals(12.5d, firstDecodedAttributes.get("double"), "double"); + assertEquals(true, firstDecodedAttributes.get("bool")); + assertEquals(asList("x", 7L, 2.5d, false), firstDecodedAttributes.get("arr")); + + V1SpanEvent secondDecodedEvent = events.get(1); + assertEquals(1234567891L, secondDecodedEvent.getTimeUnixNano()); + assertEquals("event.two", secondDecodedEvent.getName()); + assertEquals(emptyMap(), secondDecodedEvent.getAttributes()); } @Test - void testMalformedSpanEventsFallBackToEmptyEvents() throws IOException { - Map malformedEvent = Collections.singletonMap("foo", "bar"); - Map tags = Collections.singletonMap("events", malformedEvent); - - TraceGenerator.PojoSpan span = - new TraceGenerator.PojoSpan( - "service-a", - "operation-a", - "resource-a", - DDTraceId.ONE, - 123L, - 0L, - 1000L, - 2000L, - 0, - Collections.emptyMap(), - tags, - "web", - false, - PrioritySampling.SAMPLER_KEEP, - 200, - null); - - TraceMapperV1 mapper = new TraceMapperV1(); - byte[] encoded = - serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); - List stringTable = new ArrayList<>(); - stringTable.add(""); + void malformedSpanEventsFallBackToEmptyEvents() throws IOException { + // a map instead of the expected list of events + PojoSpan span = span(singletonMap(SPAN_EVENTS, singletonMap("foo", "bar"))); - List> events = readFirstSpanEvents(unpacker, stringTable); - - assertTrue(events.isEmpty()); + assertTrue(readFirstSpan(serializeV1Payload(span)).getEvents().isEmpty()); } @Test - void testMetaStructIsEncodedAsBytesAttribute() throws IOException { - TraceGenerator.PojoSpan span = - new TraceGenerator.PojoSpan( - "service-a", - "operation-a", - "resource-a", - DDTraceId.ONE, - 123L, - 0L, - 1000L, - 2000L, - 0, - Collections.emptyMap(), - Collections.emptyMap(), - "web", - false, - PrioritySampling.SAMPLER_KEEP, - 200, - null); - + void metaStructIsEncodedAsBytesAttribute() throws IOException { + PojoSpan span = span(emptyMap()); Map metaStructValue = new HashMap<>(); metaStructValue.put("foo", "bar"); metaStructValue.put("answer", 42L); span.setMetaStruct("meta_key", metaStructValue); - TraceMapperV1 mapper = new TraceMapperV1(); - byte[] encoded = - serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); - List stringTable = new ArrayList<>(); - stringTable.add(""); - - Map attributes = readFirstSpanAttributes(unpacker, stringTable); + Map attributes = readFirstSpan(serializeV1Payload(span)).getAttributes(); byte[] metaStructBytes = (byte[]) attributes.get("meta_key"); - MessageUnpacker metaStructUnpacker = MessagePack.newDefaultUnpacker(metaStructBytes); - int metaStructFieldCount = metaStructUnpacker.unpackMapHeader(); + assertNotNull(metaStructBytes); + Map decodedMetaStruct = new HashMap<>(); - for (int i = 0; i < metaStructFieldCount; i++) { - String key = metaStructUnpacker.unpackString(); - switch (metaStructUnpacker.getNextFormat().getValueType()) { - case INTEGER: - decodedMetaStruct.put(key, metaStructUnpacker.unpackLong()); - break; - case STRING: - decodedMetaStruct.put(key, metaStructUnpacker.unpackString()); - break; - default: - Assertions.fail("Unexpected meta_struct value type for key " + key); + try (MessageUnpacker metaStructUnpacker = MessagePack.newDefaultUnpacker(metaStructBytes)) { + int metaStructFieldCount = metaStructUnpacker.unpackMapHeader(); + for (int i = 0; i < metaStructFieldCount; i++) { + String key = metaStructUnpacker.unpackString(); + ValueType valueType = metaStructUnpacker.getNextFormat().getValueType(); + switch (valueType) { + case INTEGER: + decodedMetaStruct.put(key, metaStructUnpacker.unpackLong()); + break; + case STRING: + decodedMetaStruct.put(key, metaStructUnpacker.unpackString()); + break; + default: + fail("Unexpected meta_struct value type for key " + key); + } } } - assertNotNull(metaStructBytes); assertEquals("bar", decodedMetaStruct.get("foo")); assertEquals(42L, decodedMetaStruct.get("answer")); } @Test - void testMapValuedSpanTagsAreFlattenedInV1Attributes() throws IOException { - Map profile = new HashMap<>(); - profile.put("age", 30L); - - Map usr = new HashMap<>(); - usr.put("id", "123"); - usr.put("name", "alice"); - usr.put("authenticated", true); - usr.put("profile", profile); - - Map metadata0 = new HashMap<>(); - metadata0.put("event", "login"); - metadata0.put("attempts", 1L); - - Map metadata1 = new HashMap<>(); - metadata1.put("blocked", false); - - Map appsecEvents = new HashMap<>(); - appsecEvents.put("metadata0", metadata0); - appsecEvents.put("metadata1", metadata1); - + void mapValuedSpanTagsAreFlattenedInAttributes() throws IOException { + Map user = new HashMap<>(); + user.put("id", "123"); + user.put("name", "alice"); + user.put("authenticated", true); + user.put("profile", singletonMap("age", 30L)); + Map loginMetadata = new HashMap<>(); + loginMetadata.put("event", "login"); + loginMetadata.put("attempts", 1L); + Map loginSuccess = new HashMap<>(); + loginSuccess.put("metadata0", loginMetadata); + loginSuccess.put("metadata1", singletonMap("blocked", false)); Map tags = new HashMap<>(); - tags.put("usr", usr); - tags.put("appsec.events.users.login.success", appsecEvents); + tags.put("usr", user); + String loginSuccessTag = "appsec.events.users.login.success"; + tags.put(loginSuccessTag, loginSuccess); - TraceGenerator.PojoSpan span = - new TraceGenerator.PojoSpan( - "service-a", - "operation-a", - "resource-a", - DDTraceId.ONE, - 123L, - 0L, - 1000L, - 2000L, - 0, - Collections.emptyMap(), - tags, - "web", - false, - PrioritySampling.SAMPLER_KEEP, - 0, - null); - - TraceMapperV1 mapper = new TraceMapperV1(); - byte[] encoded = - serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); - List stringTable = new ArrayList<>(); - stringTable.add(""); - - Map attributes = readFirstSpanAttributes(unpacker, stringTable); - - assertTrue(attributes.containsKey("usr.id")); - assertTrue(attributes.containsKey("usr.name")); - assertTrue(attributes.containsKey("usr.authenticated")); - assertTrue(attributes.containsKey("usr.profile.age")); - assertTrue(attributes.containsKey("appsec.events.users.login.success.metadata0.event")); - assertTrue(attributes.containsKey("appsec.events.users.login.success.metadata0.attempts")); - assertTrue(attributes.containsKey("appsec.events.users.login.success.metadata1.blocked")); + // status code 0 keeps the encoder from adding an http.status_code attribute + Map attributes = + readFirstSpan(serializeV1Payload(span(tags, 0))).getAttributes(); assertEquals("123", attributes.get("usr.id")); assertEquals("alice", attributes.get("usr.name")); assertEquals(true, attributes.get("usr.authenticated")); - assertEquals(30d, ((Number) attributes.get("usr.profile.age")).doubleValue(), 0.000001d); - assertEquals("login", attributes.get("appsec.events.users.login.success.metadata0.event")); - assertEquals( - 1d, - ((Number) attributes.get("appsec.events.users.login.success.metadata0.attempts")) - .doubleValue(), - 0.000001d); - assertEquals(false, attributes.get("appsec.events.users.login.success.metadata1.blocked")); + assertAttributeValueEquals(30L, attributes.get("usr.profile.age"), "usr.profile.age"); + assertEquals("login", attributes.get(loginSuccessTag + ".metadata0.event")); + assertAttributeValueEquals( + 1L, attributes.get(loginSuccessTag + ".metadata0.attempts"), "attempts"); + assertEquals(false, attributes.get(loginSuccessTag + ".metadata1.blocked")); + // the 7 flattened entries plus thread.id and thread.name, and nothing else + assertEquals(9, attributes.size()); + // the map-valued tags themselves are replaced by their flattened entries assertFalse(attributes.containsKey("usr")); - assertFalse(attributes.containsKey("appsec.events.users.login.success")); + assertFalse(attributes.containsKey(loginSuccessTag)); } @Test - void testPrimitiveSpanTagsAreEncodedInV1Attributes() throws IOException { + void primitiveSpanTagsAreEncodedInAttributes() throws IOException { Map tags = new HashMap<>(); tags.put("tag.bool", true); tags.put("tag.int", 7); @@ -730,87 +501,43 @@ void testPrimitiveSpanTagsAreEncodedInV1Attributes() throws IOException { tags.put("tag.float", 3.5f); tags.put("tag.double", 4.25d); - TraceGenerator.PojoSpan span = - new TraceGenerator.PojoSpan( - "service-a", - "operation-a", - "resource-a", - DDTraceId.ONE, - 123L, - 0L, - 1000L, - 2000L, - 0, - Collections.emptyMap(), - tags, - "web", - false, - PrioritySampling.SAMPLER_KEEP, - 0, - null); - - TraceMapperV1 mapper = new TraceMapperV1(); - byte[] encoded = - serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); - List stringTable = new ArrayList<>(); - stringTable.add(""); - - Map attributes = readFirstSpanAttributes(unpacker, stringTable); + // status code 0 keeps the encoder from adding an http.status_code attribute + Map attributes = + readFirstSpan(serializeV1Payload(span(tags, 0))).getAttributes(); assertEquals(true, attributes.get("tag.bool")); - assertEquals(7d, ((Number) attributes.get("tag.int")).doubleValue(), 0.000001d); - assertEquals(9d, ((Number) attributes.get("tag.long")).doubleValue(), 0.000001d); - assertEquals(3.5d, ((Number) attributes.get("tag.float")).doubleValue(), 0.000001d); - assertEquals(4.25d, ((Number) attributes.get("tag.double")).doubleValue(), 0.000001d); + assertAttributeValueEquals(7, attributes.get("tag.int"), "tag.int"); + assertAttributeValueEquals(9L, attributes.get("tag.long"), "tag.long"); + assertAttributeValueEquals(3.5f, attributes.get("tag.float"), "tag.float"); + assertAttributeValueEquals(4.25d, attributes.get("tag.double"), "tag.double"); + // the 5 tags plus thread.id and thread.name, and nothing else + assertEquals(7, attributes.size()); } @Test - void testThreadMetadataIsEncodedInV1Attributes() throws IOException { - TraceGenerator.PojoSpan span = - new TraceGenerator.PojoSpan( - "service-a", - "operation-a", - "resource-a", - DDTraceId.ONE, - 123L, - 0L, - 1000L, - 2000L, - 0, - Collections.emptyMap(), - Collections.emptyMap(), - "web", - false, - PrioritySampling.SAMPLER_KEEP, - 0, - null); - - TraceMapperV1 mapper = new TraceMapperV1(); - byte[] encoded = - serializeMappedPayload(mapper, Collections.singletonList(Collections.singletonList(span))); - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded); - List stringTable = new ArrayList<>(); - stringTable.add(""); + void threadMetadataIsEncodedInAttributes() throws IOException { + // status code 0 keeps the encoder from adding an http.status_code attribute + PojoSpan span = span(emptyMap(), 0); - Map attributes = readFirstSpanAttributes(unpacker, stringTable); + Map attributes = readFirstSpan(serializeV1Payload(span)).getAttributes(); - assertAttributeValueEquals( - span.getTag(DDTags.THREAD_ID), attributes.get(DDTags.THREAD_ID), DDTags.THREAD_ID); - assertEquals(span.getTag(DDTags.THREAD_NAME).toString(), attributes.get(DDTags.THREAD_NAME)); + assertAttributeValueEquals(span.getTag(THREAD_ID), attributes.get(THREAD_ID), THREAD_ID); + Object expectedThreadName = span.getTag(THREAD_NAME); + assertEquals(expectedThreadName.toString(), attributes.get(THREAD_NAME)); + // thread metadata is all a tagless span encodes + assertEquals(2, attributes.size()); } - // --- Inner classes --- - - private static final class PayloadVerifier implements ByteBufferConsumer, WritableByteChannel { + private static final class PayloadVerifier implements ByteBufferConsumer { - private final List> expectedTraces; + private final List> expectedTraces; private final TraceMapperV1 mapper; - private ByteBuffer captured = ByteBuffer.allocate(200 << 10); + private final PayloadVerifiers.CapturingChannel channel = + new PayloadVerifiers.CapturingChannel(200 << 10); + private int position = 0; - private PayloadVerifier( - List> expectedTraces, TraceMapperV1 mapper) { + private PayloadVerifier(List> expectedTraces, TraceMapperV1 mapper) { this.expectedTraces = expectedTraces; this.mapper = mapper; } @@ -826,181 +553,49 @@ public void accept(int messageCount, ByteBuffer buffer) { } try { Payload payload = mapper.newPayload().withBody(messageCount, buffer); - payload.writeTo(this); - captured.flip(); - - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(captured); - if (messageCount == 0) { - assertEquals(0, unpacker.unpackMapHeader()); - return; - } + payload.writeTo(channel); + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(channel.flipForReading())) { + if (messageCount == 0) { + assertEquals(0, unpacker.unpackMapHeader()); + return; + } - List stringTable = new ArrayList<>(); - stringTable.add(""); - - int payloadFieldCount = unpacker.unpackMapHeader(); - assertEquals(10, payloadFieldCount); - - boolean seenChunks = false; - for (int i = 0; i < payloadFieldCount; i++) { - int fieldId = unpacker.unpackInt(); - if (fieldId == 11) { - int traceCount = unpacker.unpackArrayHeader(); - assertEquals(messageCount, traceCount); - seenChunks = true; - for (int traceIndex = 0; traceIndex < traceCount; traceIndex++) { - List expectedTrace = expectedTraces.get(position++); - verifyChunk(unpacker, expectedTrace, stringTable); + List stringTable = newStringTable(); + int payloadFieldCount = unpacker.unpackMapHeader(); + assertEquals(EXPECTED_PAYLOAD_FIELD_IDS.size(), payloadFieldCount); + + boolean seenChunks = false; + for (int i = 0; i < payloadFieldCount; i++) { + int fieldId = unpacker.unpackInt(); + if (fieldId == PayloadField.CHUNKS) { + int traceCount = unpacker.unpackArrayHeader(); + assertEquals(messageCount, traceCount); + seenChunks = true; + for (int traceIndex = 0; traceIndex < traceCount; traceIndex++) { + verifyChunk(unpacker, expectedTraces.get(position++), stringTable); + } + } else { + skipPayloadField(unpacker, fieldId, stringTable); } - } else { - skipPayloadField(unpacker, fieldId, stringTable); } - } - assertTrue(seenChunks); + assertTrue(seenChunks); + } } catch (IOException e) { - Assertions.fail(e.getMessage()); + fail(e.getMessage()); } finally { mapper.reset(); - captured.position(0); - captured.limit(captured.capacity()); - } - } - - @Override - public int write(ByteBuffer src) { - if (captured.remaining() < src.remaining()) { - ByteBuffer newBuffer = ByteBuffer.allocate(captured.capacity() + src.remaining()); - captured.flip(); - newBuffer.put(captured); - captured = newBuffer; - return write(src); + channel.resetForWriting(); } - captured.put(src); - return src.position(); } void verifyTracesConsumed() { assertEquals(expectedTraces.size(), position); } - - @Override - public boolean isOpen() { - return true; - } - - @Override - public void close() {} - } - - private static class CountingPojoSpan extends TraceGenerator.PojoSpan { - int processTagsAndBaggageCount = 0; - - CountingPojoSpan( - String serviceName, - String operationName, - CharSequence resourceName, - DDTraceId traceId, - long spanId, - long parentId, - long start, - long duration, - int error, - Map baggage, - Map tags, - CharSequence type, - boolean measured, - int samplingPriority, - int statusCode, - CharSequence origin) { - super( - serviceName, - operationName, - resourceName, - traceId, - spanId, - parentId, - start, - duration, - error, - baggage, - tags, - type, - measured, - samplingPriority, - statusCode, - origin); - } - - @Override - public void processTagsAndBaggage(MetadataConsumer consumer) { - processTagsAndBaggageCount++; - super.processTagsAndBaggage(consumer); - } - } - - private static class ByteArrayChannel implements WritableByteChannel { - private byte[] data = new byte[0]; - - @Override - public int write(ByteBuffer src) { - int len = src.remaining(); - byte[] incoming = new byte[len]; - src.get(incoming); - byte[] combined = new byte[data.length + incoming.length]; - System.arraycopy(data, 0, combined, 0, data.length); - System.arraycopy(incoming, 0, combined, data.length, incoming.length); - data = combined; - return len; - } - - byte[] bytes() { - return data; - } - - @Override - public boolean isOpen() { - return true; - } - - @Override - public void close() {} } - private static class CapturedBody implements ByteBufferConsumer { - private final TraceMapperV1 mapper; - private byte[] payloadBytes; - - private CapturedBody(TraceMapperV1 mapper) { - this.mapper = mapper; - } - - @Override - public void accept(int messageCount, ByteBuffer buffer) { - Payload payload = mapper.newPayload().withBody(messageCount, buffer); - payloadBytes = serializePayload(payload); - mapper.reset(); - } - } - - /** Subclass to expose SpanLink's protected constructor for testing. */ - private static class TestSpanLink extends SpanLink { - TestSpanLink( - DDTraceId traceId, - long spanId, - byte traceFlags, - String traceState, - SpanAttributes attributes) { - super(traceId, spanId, traceFlags, traceState, attributes); - } - } - - // --- Helper methods --- - private static void verifyChunk( - MessageUnpacker unpacker, - List expectedTrace, - List stringTable) + MessageUnpacker unpacker, List expectedTrace, List stringTable) throws IOException { int chunkFieldCount = unpacker.unpackMapHeader(); assertEquals(6, chunkFieldCount); @@ -1010,64 +605,54 @@ private static void verifyChunk( Map chunkAttributes = null; byte[] traceId = null; Integer samplingMechanism = null; - List decodedSpans = null; + boolean seenSpans = false; for (int i = 0; i < chunkFieldCount; i++) { int fieldId = unpacker.unpackInt(); switch (fieldId) { - case 1: + case ChunkField.PRIORITY: priority = unpacker.unpackInt(); break; - case 2: + case ChunkField.ORIGIN: origin = readStreamingString(unpacker, stringTable); break; - case 3: + case ChunkField.ATTRIBUTES: chunkAttributes = readAttributes(unpacker, stringTable); break; - case 4: - decodedSpans = verifySpans(unpacker, expectedTrace, stringTable); + case ChunkField.SPANS: + verifySpans(unpacker, expectedTrace, stringTable); + seenSpans = true; break; - case 6: - int traceIdLen = unpacker.unpackBinaryHeader(); - traceId = new byte[traceIdLen]; - unpacker.readPayload(traceId); + case ChunkField.TRACE_ID: + traceId = readBinary(unpacker); break; - case 7: + case ChunkField.SAMPLING_MECHANISM: samplingMechanism = unpacker.unpackInt(); break; default: - Assertions.fail("Unexpected chunk field id: " + fieldId); + fail("Unexpected chunk field id: " + fieldId); } } assertNotNull(priority); assertNotNull(origin); assertNotNull(chunkAttributes); - assertNotNull(decodedSpans); + assertTrue(seenSpans); assertNotNull(traceId); assertNotNull(samplingMechanism); - TraceGenerator.PojoSpan firstSpan = expectedTrace.get(0); - assertEquals(firstSpan.samplingPriority(), (int) priority); + PojoSpan firstSpan = expectedTrace.get(0); + assertEquals(firstSpan.samplingPriority(), priority.intValue()); assertEqualsWithNullAsEmpty(firstSpan.getOrigin(), origin); assertEquals(1, chunkAttributes.size()); assertEqualsWithNullAsEmpty( firstSpan.getLocalRootSpan().getServiceName(), (String) chunkAttributes.get("service")); assertArrayEquals(traceIdBytes(firstSpan.getTraceId()), traceId); - assertEquals(expectedSamplingMechanism(firstSpan.getTags()), (int) samplingMechanism); - } - - private static byte[] traceIdBytes(DDTraceId traceId) { - return ByteBuffer.allocate(16) - .putLong(traceId.toHighOrderLong()) - .putLong(traceId.toLong()) - .array(); + assertEquals(expectedSamplingMechanism(firstSpan.getBaggage()), samplingMechanism.intValue()); } - private static List verifySpans( - MessageUnpacker unpacker, - List expectedTrace, - List stringTable) + private static void verifySpans( + MessageUnpacker unpacker, List expectedTrace, List stringTable) throws IOException { int spanCount = unpacker.unpackArrayHeader(); assertEquals(expectedTrace.size(), spanCount); @@ -1075,23 +660,23 @@ private static List verifySpans( for (int i = 0; i < spanCount; i++) { verifySpan(unpacker, expectedTrace.get(i), stringTable); } - return expectedTrace; } private static void verifySpan( - MessageUnpacker unpacker, TraceGenerator.PojoSpan expectedSpan, List stringTable) + MessageUnpacker unpacker, PojoSpan expectedSpan, List stringTable) throws IOException { int spanFieldCount = unpacker.unpackMapHeader(); - assertEquals(16, spanFieldCount); + assertEquals(EXPECTED_SPAN_FIELD_IDS.size(), spanFieldCount); + Set spanFieldsSeen = new HashSet<>(); String service = null; String name = null; String resource = null; - Long spanId = null; - Long parentId = null; - Long start = null; - Long duration = null; - Boolean error = null; + long spanId = 0; + long parentId = 0; + long start = 0; + long duration = 0; + boolean error = false; Map attributes = null; String type = null; int linksCount = -1; @@ -1099,96 +684,101 @@ private static void verifySpan( String env = null; String version = null; String component = null; - Integer spanKind = null; + int spanKind = -1; for (int i = 0; i < spanFieldCount; i++) { int fieldId = unpacker.unpackInt(); + spanFieldsSeen.add(fieldId); switch (fieldId) { - case 1: + case SpanField.SERVICE: service = readStreamingString(unpacker, stringTable); break; - case 2: + case SpanField.NAME: name = readStreamingString(unpacker, stringTable); break; - case 3: + case SpanField.RESOURCE: resource = readStreamingString(unpacker, stringTable); break; - case 4: + case SpanField.SPAN_ID: spanId = unpackUnsignedLong(unpacker); break; - case 5: + case SpanField.PARENT_ID: parentId = unpackUnsignedLong(unpacker); break; - case 6: + case SpanField.START: start = unpacker.unpackLong(); break; - case 7: + case SpanField.DURATION: duration = unpacker.unpackLong(); break; - case 8: + case SpanField.ERROR: error = unpacker.unpackBoolean(); break; - case 9: + case SpanField.ATTRIBUTES: attributes = readAttributes(unpacker, stringTable); break; - case 10: + case SpanField.TYPE: type = readStreamingString(unpacker, stringTable); break; - case 11: + case SpanField.LINKS: linksCount = unpacker.unpackArrayHeader(); break; - case 12: + case SpanField.EVENTS: eventsCount = unpacker.unpackArrayHeader(); break; - case 13: + case SpanField.ENV: env = readStreamingString(unpacker, stringTable); break; - case 14: + case SpanField.VERSION: version = readStreamingString(unpacker, stringTable); break; - case 15: + case SpanField.COMPONENT: component = readStreamingString(unpacker, stringTable); break; - case 16: + case SpanField.KIND: spanKind = unpacker.unpackInt(); break; default: - Assertions.fail("Unexpected span field id: " + fieldId); + fail("Unexpected span field id: " + fieldId); } } + // A 16-entry map could still repeat one field id and omit another, which would leave a decoded + // value at its initial sentinel. Pinning the id set makes each field present exactly once, so + // the value assertions below cannot pass on an unwritten field (e.g. parentId 0, error false). + assertEquals(EXPECTED_SPAN_FIELD_IDS, spanFieldsSeen); + assertEqualsWithNullAsEmpty(expectedSpan.getServiceName(), service); assertEqualsWithNullAsEmpty(expectedSpan.getOperationName(), name); assertEqualsWithNullAsEmpty(expectedSpan.getResourceName(), resource); - assertEquals(expectedSpan.getSpanId(), (long) spanId); - assertEquals(expectedSpan.getParentId(), (long) parentId); - assertEquals(expectedSpan.getStartTime(), (long) start); - assertEquals(expectedSpan.getDurationNano(), (long) duration); + assertEquals(expectedSpan.getSpanId(), spanId); + assertEquals(expectedSpan.getParentId(), parentId); + assertEquals(expectedSpan.getStartTime(), start); + assertEquals(expectedSpan.getDurationNano(), duration); assertEquals(expectedSpan.getError() != 0, error); assertEqualsWithNullAsEmpty(expectedSpan.getType(), type); assertEquals(0, linksCount); assertEquals(0, eventsCount); - assertEqualsWithNullAsEmpty(expectedSpan.getTag(Tags.ENV), env); - assertEqualsWithNullAsEmpty(expectedSpan.getTag(Tags.VERSION), version); - assertEqualsWithNullAsEmpty(expectedSpan.getTag(Tags.COMPONENT), component); - assertEquals( - TraceMapperV1.getSpanKindValue(expectedSpan.getTag(Tags.SPAN_KIND)), (int) spanKind); + assertEqualsWithNullAsEmpty(expectedSpan.getTag(ENV), env); + assertEqualsWithNullAsEmpty(expectedSpan.getTag(VERSION), version); + assertEqualsWithNullAsEmpty(expectedSpan.getTag(COMPONENT), component); + assertEquals(TraceMapperV1.getSpanKindValue(expectedSpan.getTag(SPAN_KIND)), spanKind); assertNotNull(attributes); int expectedHttpStatusCode = expectedSpan.getHttpStatusCode(); boolean shouldContainHttpStatus = - expectedHttpStatusCode != 0 && !expectedSpan.getTags().containsKey("http.status_code"); + expectedHttpStatusCode != 0 && !expectedSpan.getTags().containsKey(HTTP_STATUS); Map expectedAttributes = new HashMap<>(expectedSpan.getBaggage()); - expectedAttributes.put(DDTags.THREAD_ID, expectedSpan.getTag(DDTags.THREAD_ID)); - expectedAttributes.put(DDTags.THREAD_NAME, expectedSpan.getTag(DDTags.THREAD_NAME)); + expectedAttributes.put(THREAD_ID, expectedSpan.getTag(THREAD_ID)); + expectedAttributes.put(THREAD_NAME, expectedSpan.getTag(THREAD_NAME)); for (Map.Entry entry : expectedSpan.getTags().entrySet()) { - if (DDTags.SPAN_EVENTS.equals(entry.getKey())) { + if (SPAN_EVENTS.equals(entry.getKey())) { continue; } addFlattenedExpectedAttribute(expectedAttributes, entry.getKey(), entry.getValue()); } if (shouldContainHttpStatus) { - expectedAttributes.put("http.status_code", Integer.toString(expectedHttpStatusCode)); + expectedAttributes.put(HTTP_STATUS, Integer.toString(expectedHttpStatusCode)); } if (expectedSpan.isTopLevel()) { expectedAttributes.put(InstrumentationTags.DD_TOP_LEVEL.toString(), 1d); @@ -1197,45 +787,9 @@ private static void verifySpan( assertEquals(expectedAttributes.size(), attributes.size()); for (Map.Entry entry : expectedAttributes.entrySet()) { String key = entry.getKey(); - Object expectedValue = entry.getValue(); assertTrue(attributes.containsKey(key), "Missing attribute key: " + key); - assertAttributeValueEquals(expectedValue, attributes.get(key), key); - } - } - - private static Map readAttributes( - MessageUnpacker unpacker, List stringTable) throws IOException { - int attrArraySize = unpacker.unpackArrayHeader(); - assertEquals(0, attrArraySize % 3); - int attrCount = attrArraySize / 3; - - Map attributes = new HashMap<>(); - for (int i = 0; i < attrCount; i++) { - String key = readStreamingString(unpacker, stringTable); - int attrType = unpacker.unpackInt(); - Object value; - switch (attrType) { - case TraceMapperV1.VALUE_TYPE_STRING: - value = readStreamingString(unpacker, stringTable); - break; - case TraceMapperV1.VALUE_TYPE_BOOLEAN: - value = unpacker.unpackBoolean(); - break; - case TraceMapperV1.VALUE_TYPE_FLOAT: - value = unpacker.unpackDouble(); - break; - case TraceMapperV1.VALUE_TYPE_BYTES: - int len = unpacker.unpackBinaryHeader(); - byte[] data = new byte[len]; - unpacker.readPayload(data); - value = data; - break; - default: - value = Assertions.fail("Unknown attribute value type: " + attrType); - } - attributes.put(key, value); + assertAttributeValueEquals(entry.getValue(), attributes.get(key), key); } - return attributes; } private static void assertAttributeValueEquals(Object expected, Object actual, String key) { @@ -1252,14 +806,6 @@ private static void assertAttributeValueEquals(Object expected, Object actual, S } } - private static long unpackUnsignedLong(MessageUnpacker unpacker) throws IOException { - MessageFormat format = unpacker.getNextFormat(); - if (format == MessageFormat.UINT64) { - return DDSpanId.from(unpacker.unpackBigInteger().toString()); - } - return unpacker.unpackLong(); - } - private static void addFlattenedExpectedAttribute( Map expectedAttributes, String key, Object value) { if (!(value instanceof Map)) { @@ -1272,549 +818,162 @@ private static void addFlattenedExpectedAttribute( } } - private static int expectedSamplingMechanism(Map tags) { - Object decisionMakerRaw = tags.get("_dd.p.dm"); - if (decisionMakerRaw == null) { + private static int expectedSamplingMechanism(Map propagationMetadata) { + String decisionMaker = propagationMetadata.get(DECISION_MAKER_TAG); + if (decisionMaker == null) { return SamplingMechanism.DEFAULT; } - String decisionMaker = String.valueOf(decisionMakerRaw); try { - int value = Integer.parseInt(decisionMaker); - return value < 0 ? -value : value; + return Math.abs(Integer.parseInt(decisionMaker)); } catch (NumberFormatException ignored) { int separator = decisionMaker.lastIndexOf('-'); if (separator >= 0 && separator + 1 < decisionMaker.length()) { try { - int value = Integer.parseInt(decisionMaker.substring(separator + 1)); - return value < 0 ? -value : value; + return Math.abs(Integer.parseInt(decisionMaker.substring(separator + 1))); } catch (NumberFormatException ignoredAgain) { - // fall through + // not a mechanism suffix either, fall through to the default } } return SamplingMechanism.DEFAULT; } } - private static String readStreamingString(MessageUnpacker unpacker, List stringTable) - throws IOException { - MessageFormat format = unpacker.getNextFormat(); - if (format == FIXSTR || format == STR8 || format == STR16 || format == STR32) { - String value = unpacker.unpackString(); - if (!stringTable.contains(value)) { - stringTable.add(value); - } - return value; - } - - int index = unpacker.unpackInt(); - assertTrue(index >= 0 && index < stringTable.size(), "Invalid string-table index: " + index); - return stringTable.get(index); + /** Asserts the next value is encoded as an unsigned 64-bit integer, and returns it. */ + private static long unpackUint64(MessageUnpacker unpacker) throws IOException { + assertEquals(MessageFormat.UINT64, unpacker.getNextFormat()); + return unpackUnsignedLong(unpacker); } - private static void skipPayloadField( - MessageUnpacker unpacker, int fieldId, List stringTable) throws IOException { - switch (fieldId) { - case 2: - case 3: - case 4: - case 5: - case 6: - case 7: - case 8: - case 9: - readStreamingString(unpacker, stringTable); - break; - case 10: - readAttributes(unpacker, stringTable); - break; - default: - Assertions.fail("Unexpected payload field id while skipping: " + fieldId); - } - } - - private static void skipChunkField( - MessageUnpacker unpacker, int fieldId, List stringTable) throws IOException { - switch (fieldId) { - case 1: - unpacker.unpackInt(); - break; - case 2: - readStreamingString(unpacker, stringTable); - break; - case 3: - readAttributes(unpacker, stringTable); - break; - case 4: - int spanCount = unpacker.unpackArrayHeader(); - for (int i = 0; i < spanCount; i++) { - skipSpan(unpacker, stringTable); - } - break; - case 5: - unpacker.unpackBoolean(); - break; - case 6: - int len = unpacker.unpackBinaryHeader(); - byte[] ignored = new byte[len]; - unpacker.readPayload(ignored); - break; - case 7: - unpacker.unpackInt(); - break; - default: - Assertions.fail("Unexpected chunk field id while skipping: " + fieldId); - } - } - - private static void skipSpan(MessageUnpacker unpacker, List stringTable) - throws IOException { - int fieldCount = unpacker.unpackMapHeader(); - for (int i = 0; i < fieldCount; i++) { - int fieldId = unpacker.unpackInt(); - switch (fieldId) { - case 1: - case 2: - case 3: - case 10: - case 13: - case 14: - case 15: - readStreamingString(unpacker, stringTable); - break; - case 4: - case 5: - unpacker.unpackValue().asNumberValue().toLong(); - break; - case 6: - case 7: - unpacker.unpackLong(); - break; - case 8: - unpacker.unpackBoolean(); - break; - case 9: - int attrArraySize = unpacker.unpackArrayHeader(); - int attrCount = attrArraySize / 3; - for (int j = 0; j < attrCount; j++) { - readStreamingString(unpacker, stringTable); - int type = unpacker.unpackInt(); - switch (type) { - case TraceMapperV1.VALUE_TYPE_STRING: - readStreamingString(unpacker, stringTable); - break; - case TraceMapperV1.VALUE_TYPE_BOOLEAN: - unpacker.unpackBoolean(); - break; - case TraceMapperV1.VALUE_TYPE_FLOAT: - unpacker.unpackDouble(); - break; - case TraceMapperV1.VALUE_TYPE_BYTES: - int blen = unpacker.unpackBinaryHeader(); - byte[] bignored = new byte[blen]; - unpacker.readPayload(bignored); - break; - default: - Assertions.fail("Unexpected attribute type while skipping: " + type); - } - } - break; - case 11: - case 12: - unpacker.unpackArrayHeader(); - break; - case 16: - unpacker.unpackInt(); - break; - default: - Assertions.fail("Unexpected span field id while skipping: " + fieldId); - } - } + private static byte[] serializeV1Payload(PojoSpan span) { + return serializeV1Payload(singletonList(span)); } - private static Map readFirstSpanAttributes( - MessageUnpacker unpacker, List stringTable) throws IOException { - int payloadFieldCount = unpacker.unpackMapHeader(); - for (int i = 0; i < payloadFieldCount; i++) { - int payloadFieldId = unpacker.unpackInt(); - if (payloadFieldId != 11) { - skipPayloadField(unpacker, payloadFieldId, stringTable); - continue; - } - - int chunkCount = unpacker.unpackArrayHeader(); - assertEquals(1, chunkCount); - - int chunkFieldCount = unpacker.unpackMapHeader(); - for (int chunkFieldIndex = 0; chunkFieldIndex < chunkFieldCount; chunkFieldIndex++) { - int chunkFieldId = unpacker.unpackInt(); - if (chunkFieldId != 4) { - skipChunkField(unpacker, chunkFieldId, stringTable); - continue; - } + /** Maps a single trace and returns the complete v1 payload bytes. */ + private static byte[] serializeV1Payload(List trace) { + TraceMapperV1 mapper = new TraceMapperV1(); + CapturedBody capturedBody = new CapturedBody(mapper); + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(2 << 20, capturedBody)); - int spanCount = unpacker.unpackArrayHeader(); - assertEquals(1, spanCount); + assertTrue(packer.format(trace, mapper)); + packer.flush(); - int spanFieldCount = unpacker.unpackMapHeader(); - for (int spanFieldIndex = 0; spanFieldIndex < spanFieldCount; spanFieldIndex++) { - int spanFieldId = unpacker.unpackInt(); - if (spanFieldId == 9) { - return readAttributes(unpacker, stringTable); - } - skipSpanField(unpacker, spanFieldId, stringTable); - } - } - } - return Assertions.fail("Could not find span attributes field in first span"); + assertNotNull(capturedBody.payloadBytes); + return capturedBody.payloadBytes; } - private static List> readFirstSpanLinks( - MessageUnpacker unpacker, List stringTable) throws IOException { - int payloadFieldCount = unpacker.unpackMapHeader(); - for (int i = 0; i < payloadFieldCount; i++) { - int payloadFieldId = unpacker.unpackInt(); - if (payloadFieldId != 11) { - skipPayloadField(unpacker, payloadFieldId, stringTable); - continue; - } - - int chunkCount = unpacker.unpackArrayHeader(); - assertEquals(1, chunkCount); - - int chunkFieldCount = unpacker.unpackMapHeader(); - for (int chunkFieldIndex = 0; chunkFieldIndex < chunkFieldCount; chunkFieldIndex++) { - int chunkFieldId = unpacker.unpackInt(); - if (chunkFieldId != 4) { - skipChunkField(unpacker, chunkFieldId, stringTable); - continue; - } - - int spanCount = unpacker.unpackArrayHeader(); - assertEquals(1, spanCount); - - int spanFieldCount = unpacker.unpackMapHeader(); - for (int spanFieldIndex = 0; spanFieldIndex < spanFieldCount; spanFieldIndex++) { - int spanFieldId = unpacker.unpackInt(); - if (spanFieldId == 11) { - return readSpanLinks(unpacker, stringTable); - } - skipSpanField(unpacker, spanFieldId, stringTable); - } - } - } - return Assertions.fail("Could not find span links field in first span"); + private static PojoSpan span(Map tags) { + return span(123L, 0L, emptyMap(), tags, 200, emptyList()); } - private static void skipSpanField(MessageUnpacker unpacker, int fieldId, List stringTable) - throws IOException { - switch (fieldId) { - case 1: - case 2: - case 3: - case 10: - case 13: - case 14: - case 15: - readStreamingString(unpacker, stringTable); - break; - case 4: - case 5: - unpacker.unpackValue().asNumberValue().toLong(); - break; - case 6: - case 7: - unpacker.unpackLong(); - break; - case 8: - unpacker.unpackBoolean(); - break; - case 9: - readAttributes(unpacker, stringTable); - break; - case 12: - int eventsCount = unpacker.unpackArrayHeader(); - for (int j = 0; j < eventsCount; j++) { - skipSpanEvent(unpacker, stringTable); - } - break; - case 11: - int linksCount = unpacker.unpackArrayHeader(); - for (int j = 0; j < linksCount; j++) { - int linkFieldCount = unpacker.unpackMapHeader(); - for (int k = 0; k < linkFieldCount; k++) { - int linkFieldId = unpacker.unpackInt(); - switch (linkFieldId) { - case 1: - int traceIdLen = unpacker.unpackBinaryHeader(); - byte[] traceIdIgnored = new byte[traceIdLen]; - unpacker.readPayload(traceIdIgnored); - break; - case 2: - case 5: - unpacker.unpackValue().asNumberValue().toLong(); - break; - case 3: - readAttributes(unpacker, stringTable); - break; - case 4: - readStreamingString(unpacker, stringTable); - break; - default: - Assertions.fail("Unexpected span link field id while skipping: " + linkFieldId); - } - } - } - break; - case 16: - unpacker.unpackInt(); - break; - default: - Assertions.fail("Unexpected span field id while skipping: " + fieldId); - } + private static PojoSpan span(Map tags, int statusCode) { + return span(123L, 0L, emptyMap(), tags, statusCode, emptyList()); } - private static List> readSpanLinks( - MessageUnpacker unpacker, List stringTable) throws IOException { - int linksCount = unpacker.unpackArrayHeader(); - List> links = new ArrayList<>(); - - for (int i = 0; i < linksCount; i++) { - int linkFieldCount = unpacker.unpackMapHeader(); - assertEquals(5, linkFieldCount); - - byte[] traceId = null; - Long spanId = null; - Map attributes = null; - String tracestate = null; - Long flags = null; - - for (int j = 0; j < linkFieldCount; j++) { - int linkFieldId = unpacker.unpackInt(); - switch (linkFieldId) { - case 1: - int traceIdLen = unpacker.unpackBinaryHeader(); - traceId = new byte[traceIdLen]; - unpacker.readPayload(traceId); - break; - case 2: - spanId = unpacker.unpackValue().asNumberValue().toLong(); - break; - case 3: - attributes = readAttributes(unpacker, stringTable); - break; - case 4: - tracestate = readStreamingString(unpacker, stringTable); - break; - case 5: - flags = unpacker.unpackValue().asNumberValue().toLong(); - break; - default: - Assertions.fail("Unexpected span link field id: " + linkFieldId); - } - } - - Map link = new HashMap<>(); - link.put("traceId", traceId); - link.put("spanId", spanId); - link.put("attributes", attributes); - link.put("tracestate", tracestate); - link.put("flags", flags); - links.add(link); - } - - return links; + private static PojoSpan spanWithBaggage(Map baggage) { + return span(123L, 0L, baggage, emptyMap(), 200, emptyList()); } - private static List> readFirstSpanEvents( - MessageUnpacker unpacker, List stringTable) throws IOException { - int payloadFieldCount = unpacker.unpackMapHeader(); - for (int i = 0; i < payloadFieldCount; i++) { - int payloadFieldId = unpacker.unpackInt(); - if (payloadFieldId != 11) { - skipPayloadField(unpacker, payloadFieldId, stringTable); - continue; - } - - int chunkCount = unpacker.unpackArrayHeader(); - assertEquals(1, chunkCount); - - int chunkFieldCount = unpacker.unpackMapHeader(); - for (int chunkFieldIndex = 0; chunkFieldIndex < chunkFieldCount; chunkFieldIndex++) { - int chunkFieldId = unpacker.unpackInt(); - if (chunkFieldId != 4) { - skipChunkField(unpacker, chunkFieldId, stringTable); - continue; - } - - int spanCount = unpacker.unpackArrayHeader(); - assertEquals(1, spanCount); - - int spanFieldCount = unpacker.unpackMapHeader(); - for (int spanFieldIndex = 0; spanFieldIndex < spanFieldCount; spanFieldIndex++) { - int spanFieldId = unpacker.unpackInt(); - if (spanFieldId == 12) { - return readSpanEvents(unpacker, stringTable); - } - skipSpanField(unpacker, spanFieldId, stringTable); - } - } - } - return Assertions.fail("Could not find span events field in first span"); + private static PojoSpan spanWithIds(long spanId, long parentId) { + return span(spanId, parentId, emptyMap(), emptyMap(), 200, emptyList()); } - private static List> readSpanEvents( - MessageUnpacker unpacker, List stringTable) throws IOException { - int eventsCount = unpacker.unpackArrayHeader(); - List> events = new ArrayList<>(); - - for (int i = 0; i < eventsCount; i++) { - int eventFieldCount = unpacker.unpackMapHeader(); - assertEquals(3, eventFieldCount); - - Long timeUnixNano = null; - String name = null; - Map attributes = null; - - for (int j = 0; j < eventFieldCount; j++) { - int eventFieldId = unpacker.unpackInt(); - switch (eventFieldId) { - case 1: - timeUnixNano = unpacker.unpackLong(); - break; - case 2: - name = readStreamingString(unpacker, stringTable); - break; - case 3: - attributes = readEventAttributes(unpacker, stringTable); - break; - default: - Assertions.fail("Unexpected span event field id: " + eventFieldId); - } - } - - Map event = new HashMap<>(); - event.put("timeUnixNano", timeUnixNano); - event.put("name", name); - event.put("attributes", attributes); - events.add(event); - } - return events; + private static PojoSpan spanWithLinks(List spanLinks) { + return span(123L, 0L, emptyMap(), emptyMap(), 200, spanLinks); } - private static Map readEventAttributes( - MessageUnpacker unpacker, List stringTable) throws IOException { - int attrArraySize = unpacker.unpackArrayHeader(); - assertEquals(0, attrArraySize % 3); - int attrCount = attrArraySize / 3; - Map attributes = new HashMap<>(); - - for (int i = 0; i < attrCount; i++) { - String key = readStreamingString(unpacker, stringTable); - int attrType = unpacker.unpackInt(); - Object value; - switch (attrType) { - case TraceMapperV1.VALUE_TYPE_STRING: - value = readStreamingString(unpacker, stringTable); - break; - case TraceMapperV1.VALUE_TYPE_BOOLEAN: - value = unpacker.unpackBoolean(); - break; - case TraceMapperV1.VALUE_TYPE_FLOAT: - value = unpacker.unpackDouble(); - break; - case TraceMapperV1.VALUE_TYPE_INT: - value = unpacker.unpackLong(); - break; - case TraceMapperV1.VALUE_TYPE_ARRAY: - value = readEventArrayValue(unpacker, stringTable); - break; - default: - value = Assertions.fail("Unknown event attribute value type: " + attrType); - } - attributes.put(key, value); - } - return attributes; + /** A span carrying the fields shared by most tests; only the varying pieces are parameters. */ + private static PojoSpan span( + long spanId, + long parentId, + Map baggage, + Map tags, + int statusCode, + List spanLinks) { + return new PojoSpan( + "service-a", + "operation-a", + "resource-a", + DDTraceId.ONE, + spanId, + parentId, + 1000L, + 2000L, + 0, + baggage, + tags, + "web", + false, + SAMPLER_KEEP, + statusCode, + null, + spanLinks); } - private static List readEventArrayValue( - MessageUnpacker unpacker, List stringTable) throws IOException { - int itemArraySize = unpacker.unpackArrayHeader(); - assertEquals(0, itemArraySize % 2); - int itemCount = itemArraySize / 2; - List values = new ArrayList<>(); - for (int i = 0; i < itemCount; i++) { - int itemType = unpacker.unpackInt(); - switch (itemType) { - case TraceMapperV1.VALUE_TYPE_STRING: - values.add(readStreamingString(unpacker, stringTable)); - break; - case TraceMapperV1.VALUE_TYPE_BOOLEAN: - values.add(unpacker.unpackBoolean()); - break; - case TraceMapperV1.VALUE_TYPE_FLOAT: - values.add(unpacker.unpackDouble()); - break; - case TraceMapperV1.VALUE_TYPE_INT: - values.add(unpacker.unpackLong()); - break; - default: - Assertions.fail("Unknown event array item type: " + itemType); - } + private static final class CapturedBody implements ByteBufferConsumer { + private final TraceMapperV1 mapper; + private byte[] payloadBytes; + + private CapturedBody(TraceMapperV1 mapper) { + this.mapper = mapper; } - return values; - } - private static void skipSpanEvent(MessageUnpacker unpacker, List stringTable) - throws IOException { - int fieldCount = unpacker.unpackMapHeader(); - for (int i = 0; i < fieldCount; i++) { - int fieldId = unpacker.unpackInt(); - switch (fieldId) { - case 1: - unpacker.unpackLong(); - break; - case 2: - readStreamingString(unpacker, stringTable); - break; - case 3: - readEventAttributes(unpacker, stringTable); - break; - default: - Assertions.fail("Unexpected event field id while skipping: " + fieldId); + @Override + public void accept(int messageCount, ByteBuffer buffer) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + Payload payload = mapper.newPayload().withBody(messageCount, buffer); + payload.writeTo(Channels.newChannel(out)); + payloadBytes = out.toByteArray(); + } catch (IOException e) { + fail("Failed to serialize the v1 payload: " + e.getMessage()); + } finally { + mapper.reset(); } } } - private static byte[] serializeMappedPayload( - TraceMapperV1 mapper, List> traces) { - CapturedBody capturedBody = new CapturedBody(mapper); - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(2 << 20, capturedBody)); + /** A span counting how many times its tags and baggage were handed to the mapper. */ + private static final class CountingPojoSpan extends PojoSpan { + private int processTagsAndBaggageCount = 0; - for (List trace : traces) { - assertTrue(packer.format(trace, mapper)); + private CountingPojoSpan( + String operationName, CharSequence resourceName, long spanId, long parentId) { + super( + "service-a", + operationName, + resourceName, + DDTraceId.ONE, + spanId, + parentId, + 1000L, + 2000L, + 0, + emptyMap(), + singletonMap(HTTP_URL, "http://localhost:7777/"), + "web", + false, + SAMPLER_KEEP, + 200, + null); } - packer.flush(); - assertNotNull(capturedBody.payloadBytes); - return capturedBody.payloadBytes; - } - - private static byte[] serializePayload(Payload payload) { - ByteArrayChannel channel = new ByteArrayChannel(); - try { - payload.writeTo(channel); - } catch (IOException e) { - Assertions.fail(e.getMessage()); + @Override + public void processTagsAndBaggage(MetadataConsumer consumer) { + processTagsAndBaggageCount++; + super.processTagsAndBaggage(consumer); } - return channel.bytes(); } - private static void assertEqualsWithNullAsEmpty(CharSequence expected, String actual) { - if (expected == null) { - assertEquals("", actual); - } else { - assertEquals(expected.toString(), actual); + /** {@link SpanLink}'s constructor is protected, so tests reach it through a subclass. */ + private static final class TestSpanLink extends SpanLink { + private TestSpanLink( + DDTraceId traceId, + long spanId, + byte traceFlags, + String traceState, + SpanAttributes attributes) { + super(traceId, spanId, traceFlags, traceState, attributes); } } } From b83b8cc52e46cf71e5dcf571ce0228c89d015557 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Bempel Date: Thu, 20 Aug 2026 09:38:47 +0200 Subject: [PATCH 08/10] spotless increase timeout --- .../datadog/trace/common/writer/DDAgentApiTest.java | 2 +- .../common/writer/PayloadDispatcherImplTest.java | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java index 84556c1cf9b..4e0abf5fbab 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDAgentApiTest.java @@ -40,6 +40,7 @@ import datadog.trace.core.DDSpan; import datadog.trace.core.DDSpanContext; import datadog.trace.core.propagation.PropagationTags; +import datadog.trace.test.junit.utils.tabletest.TableTestTypeConverters; import java.io.IOException; import java.lang.reflect.Field; import java.nio.ByteBuffer; @@ -55,7 +56,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; -import datadog.trace.test.junit.utils.tabletest.TableTestTypeConverters; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import org.junit.jupiter.api.Assertions; diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/PayloadDispatcherImplTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/PayloadDispatcherImplTest.java index 2bf49f69fc3..ff629171b30 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/PayloadDispatcherImplTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/PayloadDispatcherImplTest.java @@ -42,9 +42,16 @@ class PayloadDispatcherImplTest extends DDJavaSpecification { static final MonitoringImpl monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS); - // Groovy baseline: v0.5 ~5.5s, v0.4 ~1.3s; Java has higher mock overhead so use 30s timeout - @Timeout(30) - @TableTest({"scenario | traceEndpoint", "v0.5 | 'v0.5/traces'", "v0.4 | 'v0.4/traces'"}) + // Groovy baseline: v0.5 ~5.5s, v0.4 ~1.3s; + // Java has higher mock overhead, especially on first call, so use 60s timeout + @Timeout(60) + // spotless:off + @TableTest({ + "scenario | traceEndpoint", + "v0.5 | 'v0.5/traces'", + "v0.4 | 'v0.4/traces'" + }) + // spotless:on void testFlushAutomaticallyWhenDataLimitIsBreached(String traceEndpoint) throws Exception { AtomicBoolean flushed = new AtomicBoolean(); HealthMetrics healthMetrics = mock(HealthMetrics.class); From 393488e65498984d0ac99ec12267bea1af05c860 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Bempel Date: Thu, 20 Aug 2026 17:45:33 +0200 Subject: [PATCH 09/10] add try-with-resources --- .../trace/common/writer/TraceMapperTest.java | 106 +++++++++--------- .../writer/TraceProcessingWorkerTest.java | 12 +- .../writer/ddintake/DDIntakeApiTest.java | 12 +- 3 files changed, 63 insertions(+), 67 deletions(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceMapperTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceMapperTest.java index 2cf4129b664..a88d7d9000f 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceMapperTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceMapperTest.java @@ -51,62 +51,64 @@ void testTraceMapperV05() throws Exception { ByteBuffer dictionaryBytes = dictionaryBuffer.slice(); Map meta = new HashMap<>(); - MessageUnpacker dictionaryUnpacker = MessagePack.newDefaultUnpacker(dictionaryBytes); - int dictionaryLength = TraceMapperTestBridge.getEncoding(traceMapper).size(); - String[] dictionary = new String[dictionaryLength]; - for (int i = 0; i < dictionary.length; ++i) { - dictionary[i] = dictionaryUnpacker.unpackString(); - } - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(sink.captured); - int traceCount = unpacker.unpackArrayHeader(); - assertEquals(1, traceCount); - for (int i = 0; i < traceCount; ++i) { - int arrayLength = unpacker.unpackArrayHeader(); - assertEquals(12, arrayLength); - String serviceName = dictionary[unpacker.unpackInt()]; - assertEquals("my-service", serviceName); - String operationName = dictionary[unpacker.unpackInt()]; - assertTrue(operationName.isEmpty()); - String resourceName = dictionary[unpacker.unpackInt()]; - assertTrue(resourceName.isEmpty()); - long traceId = unpacker.unpackLong(); - assertTrue(traceId > 0); - long spanId = unpacker.unpackLong(); - assertTrue(spanId > 0); - long parentId = unpacker.unpackLong(); - assertEquals(0, parentId); - long start = unpacker.unpackLong(); - assertTrue(start > 0); - long duration = unpacker.unpackLong(); - assertEquals(-start, duration); - int error = unpacker.unpackInt(); - assertEquals(0, error); - int metaHeader = unpacker.unpackMapHeader(); - for (int j = 0; j < metaHeader; ++j) { - String key = dictionary[unpacker.unpackInt()]; - assertNotNull(key); - String value = dictionary[unpacker.unpackInt()]; - assertNotNull(value); - meta.put(key, value); - } - int metricsHeader = unpacker.unpackMapHeader(); - for (int j = 0; j < metricsHeader; ++j) { - String key = dictionary[unpacker.unpackInt()]; - assertNotNull(key); - unpacker.skipValue(); + try (MessageUnpacker dictionaryUnpacker = MessagePack.newDefaultUnpacker(dictionaryBytes)) { + int dictionaryLength = TraceMapperTestBridge.getEncoding(traceMapper).size(); + String[] dictionary = new String[dictionaryLength]; + for (int i = 0; i < dictionary.length; ++i) { + dictionary[i] = dictionaryUnpacker.unpackString(); } - String type = dictionary[unpacker.unpackInt()]; - assertNotNull(type); + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(sink.captured)) { + int traceCount = unpacker.unpackArrayHeader(); + assertEquals(1, traceCount); + for (int i = 0; i < traceCount; ++i) { + int arrayLength = unpacker.unpackArrayHeader(); + assertEquals(12, arrayLength); + String serviceName = dictionary[unpacker.unpackInt()]; + assertEquals("my-service", serviceName); + String operationName = dictionary[unpacker.unpackInt()]; + assertTrue(operationName.isEmpty()); + String resourceName = dictionary[unpacker.unpackInt()]; + assertTrue(resourceName.isEmpty()); + long traceId = unpacker.unpackLong(); + assertTrue(traceId > 0); + long spanId = unpacker.unpackLong(); + assertTrue(spanId > 0); + long parentId = unpacker.unpackLong(); + assertEquals(0, parentId); + long start = unpacker.unpackLong(); + assertTrue(start > 0); + long duration = unpacker.unpackLong(); + assertEquals(-start, duration); + int error = unpacker.unpackInt(); + assertEquals(0, error); + int metaHeader = unpacker.unpackMapHeader(); + for (int j = 0; j < metaHeader; ++j) { + String key = dictionary[unpacker.unpackInt()]; + assertNotNull(key); + String value = dictionary[unpacker.unpackInt()]; + assertNotNull(value); + meta.put(key, value); + } + int metricsHeader = unpacker.unpackMapHeader(); + for (int j = 0; j < metricsHeader; ++j) { + String key = dictionary[unpacker.unpackInt()]; + assertNotNull(key); + unpacker.skipValue(); + } + String type = dictionary[unpacker.unpackInt()]; + assertNotNull(type); + // find the meta entry whose key contains ".mydata." and verify its value + String myDataValue = null; + for (Map.Entry entry : meta.entrySet()) { + if (entry.getKey().contains(".mydata.")) { + myDataValue = entry.getValue(); + break; + } + } - // find the meta entry whose key contains ".mydata." and verify its value - String myDataValue = null; - for (Map.Entry entry : meta.entrySet()) { - if (entry.getKey().contains(".mydata.")) { - myDataValue = entry.getValue(); - break; + assertEquals("[1,2,3]", myDataValue); } } - assertEquals("[1,2,3]", myDataValue); } tracer.close(); diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java index 39b41fdaa44..86dbfa61b22 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java @@ -57,7 +57,7 @@ private PayloadDispatcherImpl flushCountingPayloadDispatcher(AtomicInteger flush @Test void testHeartbeatsShouldBeTriggeredAutomaticallyWhenEnabled() throws Exception { AtomicInteger flushCount = new AtomicInteger(); - TraceProcessingWorker worker = + try (TraceProcessingWorker worker = new TraceProcessingWorker( 10, mock(HealthMetrics.class), @@ -66,12 +66,11 @@ void testHeartbeatsShouldBeTriggeredAutomaticallyWhenEnabled() throws Exception FAST_LANE, 1, TimeUnit.NANOSECONDS, // stop heartbeats from being throttled - null); + null)) { - // processor is started - worker.start(); + // processor is started + worker.start(); - try { // heartbeat occurs automatically long deadline = System.currentTimeMillis() + 5000; while (System.currentTimeMillis() < deadline) { @@ -79,9 +78,6 @@ void testHeartbeatsShouldBeTriggeredAutomaticallyWhenEnabled() throws Exception Thread.sleep(50); } assertTrue(flushCount.get() > 0); - } finally { - // cleanup - worker.close(); } } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeApiTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeApiTest.java index dfc0386a2ec..c00f322d9da 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeApiTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddintake/DDIntakeApiTest.java @@ -137,7 +137,7 @@ void testRetriesWhenBackendReturns429TooManyRequests() { String apiVersion = "v2"; String path = buildIntakePath(trackType, apiVersion); int[] retry = {0}; - JavaTestHttpServer intake = + try (JavaTestHttpServer intake = JavaTestHttpServer.httpServer( s -> s.handlers( @@ -154,18 +154,16 @@ void testRetriesWhenBackendReturns429TooManyRequests() { } else { api.getResponse().status(200).send(); } - }))); - DDIntakeApi client = createIntakeApi(intake.getAddress().toString(), trackType); - Payload payload = prepareTraces(trackType, Collections.emptyList()); + })))) { + + DDIntakeApi client = createIntakeApi(intake.getAddress().toString(), trackType); + Payload payload = prepareTraces(trackType, Collections.emptyList()); - try { RemoteApi.Response clientResponse = client.sendSerializedTraces(payload); assertTrue(clientResponse.success()); assertTrue(clientResponse.status().isPresent()); assertEquals(200, clientResponse.status().getAsInt()); assertEquals(path, intake.getLastRequest().getPath()); - } finally { - intake.close(); } } From caa69aa0b243808e3a096f174a60bb8c647f9125 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Bempel Date: Thu, 20 Aug 2026 19:31:00 +0200 Subject: [PATCH 10/10] increase timeout for IBM JVMs --- .../trace/common/writer/PayloadDispatcherImplTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/PayloadDispatcherImplTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/PayloadDispatcherImplTest.java index ff629171b30..cf756003e7c 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/PayloadDispatcherImplTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/PayloadDispatcherImplTest.java @@ -42,9 +42,7 @@ class PayloadDispatcherImplTest extends DDJavaSpecification { static final MonitoringImpl monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS); - // Groovy baseline: v0.5 ~5.5s, v0.4 ~1.3s; - // Java has higher mock overhead, especially on first call, so use 60s timeout - @Timeout(60) + @Timeout(180) // spotless:off @TableTest({ "scenario | traceEndpoint",