[INS-355] Added Hashicorp vault token detector

pkg/detectors/hashicorpvaulttoken/hashicorpvaulttoken.go

@@ -0,0 +1,182 @@
+package hashicorpvaulttoken
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	regexp "github.com/wasilibs/go-re2"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	client *http.Client
+	detectors.DefaultMultiPartCredentialProvider
+	detectors.EndpointSetter
+}
+
+var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.EndpointCustomizer = (*Scanner)(nil)
+var _ detectors.CloudProvider = (*Scanner)(nil)
+
+var (
+	defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
+
+	// Vault tokens:
+	// newer vault tokens are around 90-120 chars and start with hvs. (HashiCorp Vault Service token)
+	// legacy tokens are around 18-40 chars and start with s.
+	vaultTokenPat = regexp.MustCompile(
+		`\b(hvs\.[A-Za-z0-9_-]{90,120}|s\.[A-Za-z0-9_-]{18,40})\b`,
+	)
+
+	vaultUrlPat = regexp.MustCompile(`(https?:\/\/[^\s\/]*\.hashicorp\.cloud(?::\d+)?)(?:\/[^\s]*)?`)
+)
+
+func (s Scanner) Keywords() []string {
+	return []string{"hvs.", "s."}
+}
+
+func (Scanner) CloudEndpoint() string { return "" }
+
+func (s Scanner) Description() string {
+	return "This detector detects and verifies HashiCorp Vault periodic tokens, standard service tokens and admin tokens"
+}
+
+func (s Scanner) getClient() *http.Client {
+	if s.client != nil {
+		return s.client
+	}
+	return defaultClient
+}
+
+func (s Scanner) FromData(
+	ctx context.Context,
+	verify bool,
+	data []byte,
+) (results []detectors.Result, err error) {
+
+	dataStr := string(data)
+
+	uniqueTokens := make(map[string]struct{})
+	for _, match := range vaultTokenPat.FindAllStringSubmatch(dataStr, -1) {
+		uniqueTokens[match[1]] = struct{}{}
+	}
+
+	var uniqueVaultUrls = make(map[string]struct{})
+	for _, match := range vaultUrlPat.FindAllString(dataStr, -1) {
+		url := strings.TrimSpace(match)
+		uniqueVaultUrls[url] = struct{}{}
+	}
+	endpoints := make([]string, 0, len(uniqueVaultUrls))
+	for endpoint := range uniqueVaultUrls {
+		endpoints = append(endpoints, endpoint)
+	}
+
+	for _, endpoint := range s.Endpoints(endpoints...) {
+		for token := range uniqueTokens {
+			result := detectors.Result{
+				DetectorType: detectorspb.DetectorType_HashiCorpVaultToken,
+				Raw:          []byte(token),
+				RawV2:        []byte(token + endpoint),
+				Redacted:     token[8:] + "...",
+			}
+
+			if verify {
+				verified, verificationResp, verificationErr := verifyVaultToken(
+					ctx,
+					s.getClient(),
+					endpoint,
+					token,
+				)
+				result.SetVerificationError(verificationErr, token)
+				result.Verified = verified
+				if verificationResp != nil {
+					result.ExtraData = map[string]string{
+						"policies":  strings.Join(verificationResp.Data.Policies, ", "),
+						"orphan":    fmt.Sprintf("%v", verificationResp.Data.Orphan),
+						"renewable": fmt.Sprintf("%v", verificationResp.Data.Renewable),
+						"type":      verificationResp.Data.Type,
+						"entity_id": verificationResp.Data.EntityId, // can be helpful in revoking the token
+					}
+				}
+			}
+
+			results = append(results, result)
+		}
+	}
+
+	return
+}
+
+type lookupResponse struct {
+	Data struct {
+		DisplayName string   `json:"display_name"`
+		EntityId    string   `json:"entity_id"`
+		ExpireTime  string   `json:"expire_time"`
+		Orphan      bool     `json:"orphan"`
+		Policies    []string `json:"policies"`
+		Renewable   bool     `json:"renewable"`
+		Type        string   `json:"type"`
+	}
+}
+
+func verifyVaultToken(
+	ctx context.Context,
+	client *http.Client,
+	baseUrl string,
+	token string,
+) (bool, *lookupResponse, error) {
+	url, err := url.JoinPath(baseUrl, "/v1/auth/token/lookup-self")
+	if err != nil {
+		return false, nil, err
+	}
+	req, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodGet,
+		url,
+		http.NoBody,
+	)
+	if err != nil {
+		return false, nil, err
+	}
+
+	req.Header.Set("X-Vault-Token", token)
+
+	res, err := client.Do(req)
+	if err != nil {
+		return false, nil, err
+	}
+	defer func() {
+		_, _ = io.Copy(io.Discard, res.Body)
+		_ = res.Body.Close()
+	}()
+
+	switch res.StatusCode {
+	case http.StatusOK:
+		var resp lookupResponse
+		if err := json.NewDecoder(res.Body).Decode(&resp); err != nil {
+			return false, nil, err
+		}
+
+		return true, &resp, nil
+
+	case http.StatusForbidden, http.StatusUnauthorized:
+		return false, nil, nil
+
+	default:
+		return false, nil, fmt.Errorf(
+			"unexpected HTTP response status %d",
+			res.StatusCode,
+		)
+	}
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_HashiCorpVaultToken
+}

pkg/detectors/hashicorpvaulttoken/hashicorpvaulttoken_integration_test.go

@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package hashicorpvaulttoken
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+)
+
+func TestVaultToken_FromData_Integration(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+
+	// Fetch test secrets from TruffleHog test secret storage
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors-vault")
+	if err != nil {
+		t.Fatalf("could not get test secrets: %s", err)
+	}
+
+	vaultURL := testSecrets.MustGetField("HASHICORPVAULT_CLOUD_URL")
+	// Token has maximum TTL of 32days (768h), so it should still be valid by the time this test runs
+	// but if the test fails due to an invalid token, this is the most likely culprit and the token may need to be regenerated.
+	// To regenerate the token run this command in vault web cli:
+	// write auth/token/create policies="test-policy" ttl="768h" display_name="integration-test-token"
+	token := testSecrets.MustGetField("HASHICORPVAULT_TOKEN")
+
+	fakeToken := "hvs.CAESIDdRIXSyTRAmJ2nvTohWQOPZW4gtKBeuMCQ1amIgUcWtGigKImh2cy5wdEprMjdtZWNqRXJUeElXT0lXZ0lRZVQuV2JZVlgQ3_4Q" // invalid/unused token
+
+	tests := []struct {
+		name                string
+		input               string
+		verify              bool
+		wantTokens          []string
+		wantVerified        []bool
+		wantVerificationErr bool
+	}{
+		{
+			name:   "valid token with URL, verify",
+			input:  fmt.Sprintf("%s\n%s", token, vaultURL),
+			verify: true,
+			wantTokens: []string{
+				token + vaultURL,
+			},
+			wantVerified:        []bool{true},
+			wantVerificationErr: false,
+		},
+		{
+			name:   "invalid token with URL, verify",
+			input:  fmt.Sprintf("%s\n%s", fakeToken, vaultURL),
+			verify: true,
+			wantTokens: []string{
+				fakeToken + vaultURL,
+			},
+			wantVerified:        []bool{false},
+			wantVerificationErr: false, // invalid tokens are not errors, just not verified
+		},
+		{
+			name:   "valid token with URL, no verify",
+			input:  fmt.Sprintf("%s\n%s", token, vaultURL),
+			verify: false,
+			wantTokens: []string{
+				token + vaultURL,
+			},
+			wantVerified:        []bool{false},
+			wantVerificationErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			scanner := Scanner{}
+			scanner.UseFoundEndpoints(true)
+			scanner.UseCloudEndpoint(true)
+			results, err := scanner.FromData(ctx, tt.verify, []byte(tt.input))
+			require.NoError(t, err)
+
+			if len(results) != len(tt.wantTokens) {
+				t.Fatalf("expected %d results, got %d", len(tt.wantTokens), len(results))
+			}
+
+			for i, r := range results {
+				if string(r.RawV2) != tt.wantTokens[i] && string(r.Raw) != tt.wantTokens[i] {
+					t.Errorf("expected token %s, got %s", tt.wantTokens[i], string(r.Raw))
+				}
+
+				if r.Verified != tt.wantVerified[i] {
+					t.Errorf("expected verified=%v, got %v", tt.wantVerified[i], r.Verified)
+				}
+
+				if (r.VerificationError() != nil) != tt.wantVerificationErr {
+					t.Errorf("expected verification error=%v, got %v", tt.wantVerificationErr, r.VerificationError())
+				}
+			}
+		})
+	}
+}
+
+func BenchmarkSpectralOps_FromData(b *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+
+	for name, data := range detectors.MustGetBenchmarkData() {
+		b.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}