restic/vendor/cloud.google.com/go/trace/trace_test.go

// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package trace

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"regexp"
	"strings"
	"sync"
	"testing"
	"time"

	"cloud.google.com/go/datastore"
	"cloud.google.com/go/internal/testutil"
	"cloud.google.com/go/storage"
	"golang.org/x/net/context"
	api "google.golang.org/api/cloudtrace/v1"
	compute "google.golang.org/api/compute/v1"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"
	dspb "google.golang.org/genproto/googleapis/datastore/v1"
	"google.golang.org/grpc"
)

const testProjectID = "testproject"

type fakeRoundTripper struct {
	reqc chan *http.Request
}

func newFakeRoundTripper() *fakeRoundTripper {
	return &fakeRoundTripper{reqc: make(chan *http.Request)}
}

func (rt *fakeRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
	rt.reqc <- r
	resp := &http.Response{
		Status:     "200 OK",
		StatusCode: 200,
		Body:       ioutil.NopCloser(strings.NewReader("{}")),
	}
	return resp, nil
}

func newTestClient(rt http.RoundTripper) *Client {
	t, err := NewClient(context.Background(), testProjectID, option.WithHTTPClient(&http.Client{Transport: rt}))
	if err != nil {
		panic(err)
	}
	return t
}

type fakeDatastoreServer struct {
	dspb.DatastoreServer
	fail bool
}

func (f *fakeDatastoreServer) Lookup(ctx context.Context, req *dspb.LookupRequest) (*dspb.LookupResponse, error) {
	if f.fail {
		return nil, errors.New("lookup failed")
	}
	return &dspb.LookupResponse{}, nil
}

// makeRequests makes some requests.
// span is the root span.  rt is the trace client's http client's transport.
// This is used to retrieve the trace uploaded by the client, if any.  If
// expectTrace is true, we expect a trace will be uploaded.  If synchronous is
// true, the call to Finish is expected not to return before the client has
// uploaded any traces.
func makeRequests(t *testing.T, span *Span, rt *fakeRoundTripper, synchronous bool, expectTrace bool) *http.Request {
	ctx := NewContext(context.Background(), span)
	tc := newTestClient(&noopTransport{})

	// An HTTP request.
	{
		req2, err := http.NewRequest("GET", "http://example.com/bar", nil)
		if err != nil {
			t.Fatal(err)
		}
		resp := &http.Response{StatusCode: 200}
		s := span.NewRemoteChild(req2)
		s.Finish(WithResponse(resp))
	}

	// An autogenerated API call.
	{
		rt := &fakeRoundTripper{reqc: make(chan *http.Request, 1)}
		hc := &http.Client{Transport: rt}
		computeClient, err := compute.New(hc)
		if err != nil {
			t.Fatal(err)
		}
		_, err = computeClient.Zones.List(testProjectID).Context(ctx).Do()
		if err != nil {
			t.Fatal(err)
		}
	}

	// A cloud library call that uses the autogenerated API.
	{
		rt := &fakeRoundTripper{reqc: make(chan *http.Request, 1)}
		hc := &http.Client{Transport: rt}
		storageClient, err := storage.NewClient(context.Background(), option.WithHTTPClient(hc))
		if err != nil {
			t.Fatal(err)
		}
		var objAttrsList []*storage.ObjectAttrs
		it := storageClient.Bucket("testbucket").Objects(ctx, nil)
		for {
			objAttrs, err := it.Next()
			if err != nil && err != iterator.Done {
				t.Fatal(err)
			}
			if err == iterator.Done {
				break
			}
			objAttrsList = append(objAttrsList, objAttrs)
		}
	}

	// A cloud library call that uses grpc internally.
	for _, fail := range []bool{false, true} {
		srv, err := testutil.NewServer()
		if err != nil {
			t.Fatalf("creating test datastore server: %v", err)
		}
		dspb.RegisterDatastoreServer(srv.Gsrv, &fakeDatastoreServer{fail: fail})
		srv.Start()
		conn, err := grpc.Dial(srv.Addr, grpc.WithInsecure(), grpc.WithUnaryInterceptor(tc.GRPCClientInterceptor()))
		if err != nil {
			t.Fatalf("connecting to test datastore server: %v", err)
		}
		datastoreClient, err := datastore.NewClient(ctx, testProjectID, option.WithGRPCConn(conn))
		if err != nil {
			t.Fatalf("creating datastore client: %v", err)
		}
		k := datastore.NameKey("Entity", "stringID", nil)
		e := new(datastore.Entity)
		datastoreClient.Get(ctx, k, e)
	}

	done := make(chan struct{})
	go func() {
		if synchronous {
			err := span.FinishWait()
			if err != nil {
				t.Errorf("Unexpected error from span.FinishWait: %v", err)
			}
		} else {
			span.Finish()
		}
		done <- struct{}{}
	}()
	if !expectTrace {
		<-done
		select {
		case <-rt.reqc:
			t.Errorf("Got a trace, expected none.")
		case <-time.After(5 * time.Millisecond):
		}
		return nil
	} else if !synchronous {
		<-done
		return <-rt.reqc
	} else {
		select {
		case <-done:
			t.Errorf("Synchronous Finish didn't wait for trace upload.")
			return <-rt.reqc
		case <-time.After(5 * time.Millisecond):
			r := <-rt.reqc
			<-done
			return r
		}
	}
}

func TestHeader(t *testing.T) {
	tests := []struct {
		header      string
		wantTraceID string
		wantSpanID  uint64
		wantOpts    optionFlags
		wantOK      bool
	}{
		{
			header:      "0123456789ABCDEF0123456789ABCDEF/1;o=1",
			wantTraceID: "0123456789ABCDEF0123456789ABCDEF",
			wantSpanID:  1,
			wantOpts:    1,
			wantOK:      true,
		},
		{
			header:      "0123456789ABCDEF0123456789ABCDEF/1;o=0",
			wantTraceID: "0123456789ABCDEF0123456789ABCDEF",
			wantSpanID:  1,
			wantOpts:    0,
			wantOK:      true,
		},
		{
			header:      "0123456789ABCDEF0123456789ABCDEF/1",
			wantTraceID: "0123456789ABCDEF0123456789ABCDEF",
			wantSpanID:  1,
			wantOpts:    0,
			wantOK:      true,
		},
		{
			header:      "",
			wantTraceID: "",
			wantSpanID:  0,
			wantOpts:    0,
			wantOK:      false,
		},
	}
	for _, tt := range tests {
		traceID, parentSpanID, opts, _, ok := traceInfoFromHeader(tt.header)
		if got, want := traceID, tt.wantTraceID; got != want {
			t.Errorf("TraceID(%v) = %q; want %q", tt.header, got, want)
		}
		if got, want := parentSpanID, tt.wantSpanID; got != want {
			t.Errorf("SpanID(%v) = %v; want %v", tt.header, got, want)
		}
		if got, want := opts, tt.wantOpts; got != want {
			t.Errorf("Options(%v) = %v; want %v", tt.header, got, want)
		}
		if got, want := ok, tt.wantOK; got != want {
			t.Errorf("Header exists (%v) = %v; want %v", tt.header, got, want)
		}
	}
}

func TestOutgoingReqHeader(t *testing.T) {
	all, _ := NewLimitedSampler(1, 1<<16) // trace every request

	tests := []struct {
		desc           string
		traceHeader    string
		samplingPolicy SamplingPolicy

		wantHeaderRe *regexp.Regexp
	}{
		{
			desc:           "Parent span without sampling options, client samples all",
			traceHeader:    "0123456789ABCDEF0123456789ABCDEF/1",
			samplingPolicy: all,
			wantHeaderRe:   regexp.MustCompile("0123456789ABCDEF0123456789ABCDEF/\\d+;o=1"),
		},
		{
			desc:           "Parent span without sampling options, without client sampling",
			traceHeader:    "0123456789ABCDEF0123456789ABCDEF/1",
			samplingPolicy: nil,
			wantHeaderRe:   regexp.MustCompile("0123456789ABCDEF0123456789ABCDEF/\\d+;o=0"),
		},
		{
			desc:           "Parent span with o=1, client samples none",
			traceHeader:    "0123456789ABCDEF0123456789ABCDEF/1;o=1",
			samplingPolicy: nil,
			wantHeaderRe:   regexp.MustCompile("0123456789ABCDEF0123456789ABCDEF/\\d+;o=1"),
		},
		{
			desc:           "Parent span with o=0, without client sampling",
			traceHeader:    "0123456789ABCDEF0123456789ABCDEF/1;o=0",
			samplingPolicy: nil,
			wantHeaderRe:   regexp.MustCompile("0123456789ABCDEF0123456789ABCDEF/\\d+;o=0"),
		},
	}

	tc := newTestClient(nil)
	for _, tt := range tests {
		tc.SetSamplingPolicy(tt.samplingPolicy)
		span := tc.SpanFromHeader("/foo", tt.traceHeader)

		req, _ := http.NewRequest("GET", "http://localhost", nil)
		span.NewRemoteChild(req)

		if got, re := req.Header.Get(httpHeader), tt.wantHeaderRe; !re.MatchString(got) {
			t.Errorf("%v (parent=%q): got header %q; want in format %q", tt.desc, tt.traceHeader, got, re)
		}
	}
}

func TestTrace(t *testing.T) {
	t.Parallel()
	testTrace(t, false, true)
}

func TestTraceWithWait(t *testing.T) {
	testTrace(t, true, true)
}

func TestTraceFromHeader(t *testing.T) {
	t.Parallel()
	testTrace(t, false, false)
}

func TestTraceFromHeaderWithWait(t *testing.T) {
	testTrace(t, false, true)
}

func TestNewSpan(t *testing.T) {
	t.Skip("flaky")
	const traceID = "0123456789ABCDEF0123456789ABCDEF"

	rt := newFakeRoundTripper()
	traceClient := newTestClient(rt)
	span := traceClient.NewSpan("/foo")
	span.trace.traceID = traceID

	uploaded := makeRequests(t, span, rt, true, true)

	if uploaded == nil {
		t.Fatalf("No trace uploaded, expected one.")
	}

	expected := api.Traces{
		Traces: []*api.Trace{
			{
				ProjectId: testProjectID,
				Spans: []*api.TraceSpan{
					{
						Kind: "RPC_CLIENT",
						Labels: map[string]string{
							"trace.cloud.google.com/http/host":        "example.com",
							"trace.cloud.google.com/http/method":      "GET",
							"trace.cloud.google.com/http/status_code": "200",
							"trace.cloud.google.com/http/url":         "http://example.com/bar",
						},
						Name: "example.com/bar",
					},
					{
						Kind: "RPC_CLIENT",
						Labels: map[string]string{
							"trace.cloud.google.com/http/host":        "www.googleapis.com",
							"trace.cloud.google.com/http/method":      "GET",
							"trace.cloud.google.com/http/status_code": "200",
							"trace.cloud.google.com/http/url":         "https://www.googleapis.com/compute/v1/projects/testproject/zones",
						},
						Name: "www.googleapis.com/compute/v1/projects/testproject/zones",
					},
					{
						Kind: "RPC_CLIENT",
						Labels: map[string]string{
							"trace.cloud.google.com/http/host":        "www.googleapis.com",
							"trace.cloud.google.com/http/method":      "GET",
							"trace.cloud.google.com/http/status_code": "200",
							"trace.cloud.google.com/http/url":         "https://www.googleapis.com/storage/v1/b/testbucket/o",
						},
						Name: "www.googleapis.com/storage/v1/b/testbucket/o",
					},
					&api.TraceSpan{
						Kind:   "RPC_CLIENT",
						Labels: nil,
						Name:   "/google.datastore.v1.Datastore/Lookup",
					},
					&api.TraceSpan{
						Kind:   "RPC_CLIENT",
						Labels: map[string]string{"error": "rpc error: code = Unknown desc = lookup failed"},
						Name:   "/google.datastore.v1.Datastore/Lookup",
					},
					{
						Kind:   "SPAN_KIND_UNSPECIFIED",
						Labels: map[string]string{},
						Name:   "/foo",
					},
				},
				TraceId: traceID,
			},
		},
	}

	body, err := ioutil.ReadAll(uploaded.Body)
	if err != nil {
		t.Fatal(err)
	}
	var patch api.Traces
	err = json.Unmarshal(body, &patch)
	if err != nil {
		t.Fatal(err)
	}

	checkTraces(t, patch, expected)

	n := len(patch.Traces[0].Spans)
	rootSpan := patch.Traces[0].Spans[n-1]
	for i, s := range patch.Traces[0].Spans {
		if a, b := s.StartTime, s.EndTime; a > b {
			t.Errorf("span %d start time is later than its end time (%q, %q)", i, a, b)
		}
		if a, b := rootSpan.StartTime, s.StartTime; a > b {
			t.Errorf("trace start time is later than span %d start time (%q, %q)", i, a, b)
		}
		if a, b := s.EndTime, rootSpan.EndTime; a > b {
			t.Errorf("span %d end time is later than trace end time (%q, %q)", i, a, b)
		}
		if i > 1 && i < n-1 {
			if a, b := patch.Traces[0].Spans[i-1].EndTime, s.StartTime; a > b {
				t.Errorf("span %d end time is later than span %d start time (%q, %q)", i-1, i, a, b)
			}
		}
	}

	if x := rootSpan.ParentSpanId; x != 0 {
		t.Errorf("Incorrect ParentSpanId: got %d want %d", x, 0)
	}
	for i, s := range patch.Traces[0].Spans {
		if x, y := rootSpan.SpanId, s.ParentSpanId; i < n-1 && x != y {
			t.Errorf("Incorrect ParentSpanId in span %d: got %d want %d", i, y, x)
		}
	}
	for i, s := range patch.Traces[0].Spans {
		s.EndTime = ""
		labels := &expected.Traces[0].Spans[i].Labels
		for key, value := range *labels {
			if v, ok := s.Labels[key]; !ok {
				t.Errorf("Span %d is missing Label %q:%q", i, key, value)
			} else if key == "trace.cloud.google.com/http/url" {
				if !strings.HasPrefix(v, value) {
					t.Errorf("Span %d Label %q: got value %q want prefix %q", i, key, v, value)
				}
			} else if v != value {
				t.Errorf("Span %d Label %q: got value %q want %q", i, key, v, value)
			}
		}
		for key := range s.Labels {
			if _, ok := (*labels)[key]; key != "trace.cloud.google.com/stacktrace" && !ok {
				t.Errorf("Span %d: unexpected label %q", i, key)
			}
		}
		*labels = nil
		s.Labels = nil
		s.ParentSpanId = 0
		if s.SpanId == 0 {
			t.Errorf("Incorrect SpanId: got 0 want nonzero")
		}
		s.SpanId = 0
		s.StartTime = ""
	}
	if !testutil.Equal(patch, expected) {
		got, _ := json.Marshal(patch)
		want, _ := json.Marshal(expected)
		t.Errorf("PatchTraces request: got %s want %s", got, want)
	}
}

func testTrace(t *testing.T, synchronous bool, fromRequest bool) {
	t.Skip("flaky")
	const header = `0123456789ABCDEF0123456789ABCDEF/42;o=3`
	rt := newFakeRoundTripper()
	traceClient := newTestClient(rt)

	span := traceClient.SpanFromHeader("/foo", header)
	headerOrReqLabels := map[string]string{}
	headerOrReqName := "/foo"

	if fromRequest {
		req, err := http.NewRequest("GET", "http://example.com/foo", nil)
		if err != nil {
			t.Fatal(err)
		}
		req.Header.Set("X-Cloud-Trace-Context", header)
		span = traceClient.SpanFromRequest(req)
		headerOrReqLabels = map[string]string{
			"trace.cloud.google.com/http/host":   "example.com",
			"trace.cloud.google.com/http/method": "GET",
			"trace.cloud.google.com/http/url":    "http://example.com/foo",
		}
		headerOrReqName = "example.com/foo"
	}

	uploaded := makeRequests(t, span, rt, synchronous, true)
	if uploaded == nil {
		t.Fatalf("No trace uploaded, expected one.")
	}

	expected := api.Traces{
		Traces: []*api.Trace{
			{
				ProjectId: testProjectID,
				Spans: []*api.TraceSpan{
					{
						Kind: "RPC_CLIENT",
						Labels: map[string]string{
							"trace.cloud.google.com/http/host":        "example.com",
							"trace.cloud.google.com/http/method":      "GET",
							"trace.cloud.google.com/http/status_code": "200",
							"trace.cloud.google.com/http/url":         "http://example.com/bar",
						},
						Name: "example.com/bar",
					},
					{
						Kind: "RPC_CLIENT",
						Labels: map[string]string{
							"trace.cloud.google.com/http/host":        "www.googleapis.com",
							"trace.cloud.google.com/http/method":      "GET",
							"trace.cloud.google.com/http/status_code": "200",
							"trace.cloud.google.com/http/url":         "https://www.googleapis.com/compute/v1/projects/testproject/zones",
						},
						Name: "www.googleapis.com/compute/v1/projects/testproject/zones",
					},
					{
						Kind: "RPC_CLIENT",
						Labels: map[string]string{
							"trace.cloud.google.com/http/host":        "www.googleapis.com",
							"trace.cloud.google.com/http/method":      "GET",
							"trace.cloud.google.com/http/status_code": "200",
							"trace.cloud.google.com/http/url":         "https://www.googleapis.com/storage/v1/b/testbucket/o",
						},
						Name: "www.googleapis.com/storage/v1/b/testbucket/o",
					},
					&api.TraceSpan{
						Kind:   "RPC_CLIENT",
						Labels: nil,
						Name:   "/google.datastore.v1.Datastore/Lookup",
					},
					&api.TraceSpan{
						Kind:   "RPC_CLIENT",
						Labels: map[string]string{"error": "rpc error: code = Unknown desc = lookup failed"},
						Name:   "/google.datastore.v1.Datastore/Lookup",
					},
					{
						Kind:   "RPC_SERVER",
						Labels: headerOrReqLabels,
						Name:   headerOrReqName,
					},
				},
				TraceId: "0123456789ABCDEF0123456789ABCDEF",
			},
		},
	}

	body, err := ioutil.ReadAll(uploaded.Body)
	if err != nil {
		t.Fatal(err)
	}
	var patch api.Traces
	err = json.Unmarshal(body, &patch)
	if err != nil {
		t.Fatal(err)
	}

	checkTraces(t, patch, expected)

	n := len(patch.Traces[0].Spans)
	rootSpan := patch.Traces[0].Spans[n-1]
	for i, s := range patch.Traces[0].Spans {
		if a, b := s.StartTime, s.EndTime; a > b {
			t.Errorf("span %d start time is later than its end time (%q, %q)", i, a, b)
		}
		if a, b := rootSpan.StartTime, s.StartTime; a > b {
			t.Errorf("trace start time is later than span %d start time (%q, %q)", i, a, b)
		}
		if a, b := s.EndTime, rootSpan.EndTime; a > b {
			t.Errorf("span %d end time is later than trace end time (%q, %q)", i, a, b)
		}
		if i > 1 && i < n-1 {
			if a, b := patch.Traces[0].Spans[i-1].EndTime, s.StartTime; a > b {
				t.Errorf("span %d end time is later than span %d start time (%q, %q)", i-1, i, a, b)
			}
		}
	}

	if x := rootSpan.ParentSpanId; x != 42 {
		t.Errorf("Incorrect ParentSpanId: got %d want %d", x, 42)
	}
	for i, s := range patch.Traces[0].Spans {
		if x, y := rootSpan.SpanId, s.ParentSpanId; i < n-1 && x != y {
			t.Errorf("Incorrect ParentSpanId in span %d: got %d want %d", i, y, x)
		}
	}
	for i, s := range patch.Traces[0].Spans {
		s.EndTime = ""
		labels := &expected.Traces[0].Spans[i].Labels
		for key, value := range *labels {
			if v, ok := s.Labels[key]; !ok {
				t.Errorf("Span %d is missing Label %q:%q", i, key, value)
			} else if key == "trace.cloud.google.com/http/url" {
				if !strings.HasPrefix(v, value) {
					t.Errorf("Span %d Label %q: got value %q want prefix %q", i, key, v, value)
				}
			} else if v != value {
				t.Errorf("Span %d Label %q: got value %q want %q", i, key, v, value)
			}
		}
		for key := range s.Labels {
			if _, ok := (*labels)[key]; key != "trace.cloud.google.com/stacktrace" && !ok {
				t.Errorf("Span %d: unexpected label %q", i, key)
			}
		}
		*labels = nil
		s.Labels = nil
		s.ParentSpanId = 0
		if s.SpanId == 0 {
			t.Errorf("Incorrect SpanId: got 0 want nonzero")
		}
		s.SpanId = 0
		s.StartTime = ""
	}
	if !testutil.Equal(patch, expected) {
		got, _ := json.Marshal(patch)
		want, _ := json.Marshal(expected)
		t.Errorf("PatchTraces request: got %s \n\n want %s", got, want)
	}
}

func TestNoTrace(t *testing.T) {
	testNoTrace(t, false, true)
}

func TestNoTraceWithWait(t *testing.T) {
	testNoTrace(t, true, true)
}

func TestNoTraceFromHeader(t *testing.T) {
	testNoTrace(t, false, false)
}

func TestNoTraceFromHeaderWithWait(t *testing.T) {
	testNoTrace(t, true, false)
}

func testNoTrace(t *testing.T, synchronous bool, fromRequest bool) {
	for _, header := range []string{
		`0123456789ABCDEF0123456789ABCDEF/42;o=2`,
		`0123456789ABCDEF0123456789ABCDEF/42;o=0`,
		`0123456789ABCDEF0123456789ABCDEF/42`,
		`0123456789ABCDEF0123456789ABCDEF`,
		``,
	} {
		rt := newFakeRoundTripper()
		traceClient := newTestClient(rt)
		var span *Span
		if fromRequest {
			req, err := http.NewRequest("GET", "http://example.com/foo", nil)
			if header != "" {
				req.Header.Set("X-Cloud-Trace-Context", header)
			}
			if err != nil {
				t.Fatal(err)
			}
			span = traceClient.SpanFromRequest(req)
		} else {
			span = traceClient.SpanFromHeader("/foo", header)
		}
		uploaded := makeRequests(t, span, rt, synchronous, false)
		if uploaded != nil {
			t.Errorf("Got a trace, expected none.")
		}
	}
}

func TestSample(t *testing.T) {
	// A deterministic test of the sampler logic.
	type testCase struct {
		rate   float64
		maxqps float64
		want   int
	}
	const delta = 25 * time.Millisecond
	for _, test := range []testCase{
		// qps won't matter, so we will sample half of the 79 calls
		{0.50, 100, 40},
		// with 1 qps and a burst of 2, we will sample twice in second #1, once in the partial second #2
		{0.50, 1, 3},
	} {
		sp, err := NewLimitedSampler(test.rate, test.maxqps)
		if err != nil {
			t.Fatal(err)
		}
		s := sp.(*sampler)
		sampled := 0
		tm := time.Now()
		for i := 0; i < 80; i++ {
			if s.sample(Parameters{}, tm, float64(i%2)).Sample {
				sampled++
			}
			tm = tm.Add(delta)
		}
		if sampled != test.want {
			t.Errorf("rate=%f, maxqps=%f: got %d samples, want %d", test.rate, test.maxqps, sampled, test.want)
		}
	}
}

func TestSampling(t *testing.T) {
	t.Parallel()
	// This scope tests sampling in a larger context, with real time and randomness.
	wg := sync.WaitGroup{}
	type testCase struct {
		rate          float64
		maxqps        float64
		expectedRange [2]int
	}
	for _, test := range []testCase{
		{0, 5, [2]int{0, 0}},
		{5, 0, [2]int{0, 0}},
		{0.50, 100, [2]int{20, 60}},
		{0.50, 1, [2]int{3, 4}}, // Windows, with its less precise clock, sometimes gives 4.
	} {
		wg.Add(1)
		go func(test testCase) {
			rt := newFakeRoundTripper()
			traceClient := newTestClient(rt)
			traceClient.bundler.BundleByteLimit = 1
			p, err := NewLimitedSampler(test.rate, test.maxqps)
			if err != nil {
				t.Fatalf("NewLimitedSampler: %v", err)
			}
			traceClient.SetSamplingPolicy(p)
			ticker := time.NewTicker(25 * time.Millisecond)
			sampled := 0
			for i := 0; i < 79; i++ {
				req, err := http.NewRequest("GET", "http://example.com/foo", nil)
				if err != nil {
					t.Fatal(err)
				}
				span := traceClient.SpanFromRequest(req)
				span.Finish()
				select {
				case <-rt.reqc:
					<-ticker.C
					sampled++
				case <-ticker.C:
				}
			}
			ticker.Stop()
			if test.expectedRange[0] > sampled || sampled > test.expectedRange[1] {
				t.Errorf("rate=%f, maxqps=%f: got %d samples want ∈ %v", test.rate, test.maxqps, sampled, test.expectedRange)
			}
			wg.Done()
		}(test)
	}
	wg.Wait()
}

func TestBundling(t *testing.T) {
	t.Parallel()
	rt := newFakeRoundTripper()
	traceClient := newTestClient(rt)
	traceClient.bundler.DelayThreshold = time.Second / 2
	traceClient.bundler.BundleCountThreshold = 10
	p, err := NewLimitedSampler(1, 99) // sample every request.
	if err != nil {
		t.Fatalf("NewLimitedSampler: %v", err)
	}
	traceClient.SetSamplingPolicy(p)

	for i := 0; i < 35; i++ {
		go func() {
			req, err := http.NewRequest("GET", "http://example.com/foo", nil)
			if err != nil {
				t.Fatal(err)
			}
			span := traceClient.SpanFromRequest(req)
			span.Finish()
		}()
	}

	// Read the first three bundles.
	<-rt.reqc
	<-rt.reqc
	<-rt.reqc

	// Test that the fourth bundle isn't sent early.
	select {
	case <-rt.reqc:
		t.Errorf("bundle sent too early")
	case <-time.After(time.Second / 4):
		<-rt.reqc
	}

	// Test that there aren't extra bundles.
	select {
	case <-rt.reqc:
		t.Errorf("too many bundles sent")
	case <-time.After(time.Second):
	}
}

func TestWeights(t *testing.T) {
	const (
		expectedNumTraced   = 10100
		numTracedEpsilon    = 100
		expectedTotalWeight = 50000
		totalWeightEpsilon  = 5000
	)
	rng := rand.New(rand.NewSource(1))
	const delta = 2 * time.Millisecond
	for _, headerRate := range []float64{0.0, 0.5, 1.0} {
		// Simulate 10 seconds of requests arriving at 500qps.
		//
		// The sampling policy tries to sample 25% of them, but has a qps limit of
		// 100, so it will not be able to.  The returned weight should be higher
		// for some sampled requests to compensate.
		//
		// headerRate is the fraction of incoming requests that have a trace header
		// set.  The qps limit should not be exceeded, even if headerRate is high.
		sp, err := NewLimitedSampler(0.25, 100)
		if err != nil {
			t.Fatal(err)
		}
		s := sp.(*sampler)
		tm := time.Now()
		totalWeight := 0.0
		numTraced := 0
		seenLargeWeight := false
		for i := 0; i < 50000; i++ {
			d := s.sample(Parameters{HasTraceHeader: rng.Float64() < headerRate}, tm, rng.Float64())
			if d.Trace {
				numTraced++
			}
			if d.Sample {
				totalWeight += d.Weight
				if x := int(d.Weight) / 4; x <= 0 || x >= 100 || d.Weight != float64(x)*4.0 {
					t.Errorf("weight: got %f, want a small positive multiple of 4", d.Weight)
				}
				if d.Weight > 4 {
					seenLargeWeight = true
				}
			}
			tm = tm.Add(delta)
		}
		if !seenLargeWeight {
			t.Errorf("headerRate %f: never saw sample weight higher than 4.", headerRate)
		}
		if numTraced < expectedNumTraced-numTracedEpsilon || expectedNumTraced+numTracedEpsilon < numTraced {
			t.Errorf("headerRate %f: got %d traced requests, want ∈ [%d, %d]", headerRate, numTraced, expectedNumTraced-numTracedEpsilon, expectedNumTraced+numTracedEpsilon)
		}
		if totalWeight < expectedTotalWeight-totalWeightEpsilon || expectedTotalWeight+totalWeightEpsilon < totalWeight {
			t.Errorf("headerRate %f: got total weight %f want ∈ [%d, %d]", headerRate, totalWeight, expectedTotalWeight-totalWeightEpsilon, expectedTotalWeight+totalWeightEpsilon)
		}
	}
}

type alwaysTrace struct{}

func (a alwaysTrace) Sample(p Parameters) Decision {
	return Decision{Trace: true}
}

type neverTrace struct{}

func (a neverTrace) Sample(p Parameters) Decision {
	return Decision{Trace: false}
}

func TestPropagation(t *testing.T) {
	rt := newFakeRoundTripper()
	traceClient := newTestClient(rt)
	for _, header := range []string{
		`0123456789ABCDEF0123456789ABCDEF/42;o=0`,
		`0123456789ABCDEF0123456789ABCDEF/42;o=1`,
		`0123456789ABCDEF0123456789ABCDEF/42;o=2`,
		`0123456789ABCDEF0123456789ABCDEF/42;o=3`,
		`0123456789ABCDEF0123456789ABCDEF/0;o=0`,
		`0123456789ABCDEF0123456789ABCDEF/0;o=1`,
		`0123456789ABCDEF0123456789ABCDEF/0;o=2`,
		`0123456789ABCDEF0123456789ABCDEF/0;o=3`,
		``,
	} {
		for _, policy := range []SamplingPolicy{
			nil,
			alwaysTrace{},
			neverTrace{},
		} {
			traceClient.SetSamplingPolicy(policy)
			req, err := http.NewRequest("GET", "http://example.com/foo", nil)
			if err != nil {
				t.Fatal(err)
			}
			if header != "" {
				req.Header.Set("X-Cloud-Trace-Context", header)
			}

			span := traceClient.SpanFromRequest(req)

			req2, err := http.NewRequest("GET", "http://example.com/bar", nil)
			if err != nil {
				t.Fatal(err)
			}
			req3, err := http.NewRequest("GET", "http://example.com/baz", nil)
			if err != nil {
				t.Fatal(err)
			}
			span.NewRemoteChild(req2)
			span.NewRemoteChild(req3)

			var (
				t1, t2, t3 string
				s1, s2, s3 uint64
				o1, o2, o3 uint64
			)
			fmt.Sscanf(header, "%32s/%d;o=%d", &t1, &s1, &o1)
			fmt.Sscanf(req2.Header.Get("X-Cloud-Trace-Context"), "%32s/%d;o=%d", &t2, &s2, &o2)
			fmt.Sscanf(req3.Header.Get("X-Cloud-Trace-Context"), "%32s/%d;o=%d", &t3, &s3, &o3)

			if header == "" {
				if t2 != t3 {
					t.Errorf("expected the same trace ID in child requests, got %q %q", t2, t3)
				}
			} else {
				if t2 != t1 || t3 != t1 {
					t.Errorf("trace IDs should be passed to child requests")
				}
			}
			trace := policy == alwaysTrace{} || policy == nil && (o1&1) != 0
			if header == "" {
				if trace && (s2 == 0 || s3 == 0) {
					t.Errorf("got span IDs %d %d in child requests, want nonzero", s2, s3)
				}
				if trace && s2 == s3 {
					t.Errorf("got span IDs %d %d in child requests, should be different", s2, s3)
				}
				if !trace && (s2 != 0 || s3 != 0) {
					t.Errorf("got span IDs %d %d in child requests, want zero", s2, s3)
				}
			} else {
				if trace && (s2 == s1 || s3 == s1 || s2 == s3) {
					t.Errorf("parent span IDs in input and outputs should be all different, got %d %d %d", s1, s2, s3)
				}
				if !trace && (s2 != s1 || s3 != s1) {
					t.Errorf("parent span ID in input, %d, should have been equal to parent span IDs in output: %d %d", s1, s2, s3)
				}
			}
			expectTraceOption := policy == alwaysTrace{} || (o1&1) != 0
			if expectTraceOption != ((o2&1) != 0) || expectTraceOption != ((o3&1) != 0) {
				t.Errorf("tracing flag in child requests should be %t, got options %d %d", expectTraceOption, o2, o3)
			}
		}
	}
}

func BenchmarkSpanFromHeader(b *testing.B) {
	const header = `0123456789ABCDEF0123456789ABCDEF/42;o=0`
	const name = "/foo"

	rt := newFakeRoundTripper()
	traceClient := newTestClient(rt)
	for n := 0; n < b.N; n++ {
		traceClient.SpanFromHeader(name, header)
	}
}

func checkTraces(t *testing.T, patch, expected api.Traces) {
	if len(patch.Traces) != len(expected.Traces) || len(patch.Traces[0].Spans) != len(expected.Traces[0].Spans) {
		diff := testutil.Diff(patch.Traces, expected.Traces)
		t.Logf("diff:\n%s", diff)
		got, _ := json.Marshal(patch)
		want, _ := json.Marshal(expected)
		t.Fatalf("PatchTraces request: got %s want %s", got, want)
	}
}