From 252be3eaff2d72e9e892160a42dd77ae77bd3e3a Mon Sep 17 00:00:00 2001 From: Mario Corchero Date: Mon, 30 Mar 2026 17:48:34 +0200 Subject: [PATCH] feat: add host, db and username to ExtraData for database detectors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Populate ExtraData with parsed fields for all database connection string detectors (MongoDB, PostgreSQL, Redis, JDBC). This surfaces useful metadata about detected credentials. The parsing logic already existed in each detector — this change exposes the extracted values in the result's ExtraData map alongside any pre-existing fields (rotation_guide, sslmode, etc.). --- pkg/detectors/jdbc/jdbc.go | 51 +++++++++------ pkg/detectors/jdbc/jdbc_test.go | 86 +++++++++++++++++++++++++ pkg/detectors/mongodb/mongodb.go | 29 +++++++-- pkg/detectors/mongodb/mongodb_test.go | 81 +++++++++++++++++++++++ pkg/detectors/postgres/postgres.go | 13 ++++ pkg/detectors/postgres/postgres_test.go | 57 ++++++++++++++++ pkg/detectors/redis/redis.go | 13 ++++ pkg/detectors/redis/redis_test.go | 53 +++++++++++++++ 8 files changed, 358 insertions(+), 25 deletions(-) diff --git a/pkg/detectors/jdbc/jdbc.go b/pkg/detectors/jdbc/jdbc.go index e0a249409101..591c79fd67aa 100644 --- a/pkg/detectors/jdbc/jdbc.go +++ b/pkg/detectors/jdbc/jdbc.go @@ -85,27 +85,42 @@ matchLoop: Redacted: tryRedactAnonymousJDBC(jdbcConn), } - if verify { - j, err := NewJDBC(logCtx, jdbcConn) - if err != nil { - continue + // Try to parse connection info for ExtraData regardless of verification. + if j, parseErr := NewJDBC(logCtx, jdbcConn); parseErr == nil { + if info := j.GetConnectionInfo(); info != nil { + extraData := make(map[string]string) + if info.Host != "" { + extraData["host"] = info.Host + } + if info.User != "" { + extraData["username"] = info.User + } + if info.Database != "" { + extraData["database"] = info.Database + } + if len(extraData) > 0 { + result.ExtraData = extraData + } } - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - pingRes := j.ping(ctx) - result.Verified = pingRes.err == nil - // If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the - // behavior before tri-state verification was introduced and preserving it allows us to gradually migrate - // detectors to use tri-state verification. - if pingRes.err != nil && !pingRes.determinate { - err = pingRes.err - result.SetVerificationError(err, jdbcConn) - } - result.AnalysisInfo = map[string]string{ - "connection_string": jdbcConn, + if verify { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + pingRes := j.ping(ctx) + result.Verified = pingRes.err == nil + // If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the + // behavior before tri-state verification was introduced and preserving it allows us to gradually migrate + // detectors to use tri-state verification. + if pingRes.err != nil && !pingRes.determinate { + result.SetVerificationError(pingRes.err, jdbcConn) + } + result.AnalysisInfo = map[string]string{ + "connection_string": jdbcConn, + } + // TODO: specialized redaction } - // TODO: specialized redaction + } else if verify { + continue } results = append(results, result) diff --git a/pkg/detectors/jdbc/jdbc_test.go b/pkg/detectors/jdbc/jdbc_test.go index 2ba0c937a95e..2e0ad29962df 100644 --- a/pkg/detectors/jdbc/jdbc_test.go +++ b/pkg/detectors/jdbc/jdbc_test.go @@ -151,6 +151,92 @@ func TestJdbc_Pattern(t *testing.T) { } } +func TestJdbc_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + wantDatabase string + }{ + { + name: "mysql with basic auth", + data: `jdbc:mysql://root:password@localhost:3306/testdb`, + wantHost: "tcp(localhost:3306)", + wantUsername: "root", + wantDatabase: "testdb", + }, + { + name: "postgresql with basic auth", + data: `jdbc:postgresql://postgres:secret@dbhost:5432/mydb`, + wantHost: "dbhost:5432", + wantUsername: "postgres", + wantDatabase: "mydb", + }, + { + name: "sqlserver with semicolon params", + data: `jdbc:sqlserver://server.example.com:1433;database=testdb;user=sa;password=Pass123`, + wantHost: "server.example.com:1433", + wantUsername: "sa", + wantDatabase: "testdb", + }, + { + name: "mysql with query params for credentials", + data: `jdbc:mysql://dbhost:3307/testdb?user=admin&password=secret`, + wantHost: "tcp(dbhost:3307)", + wantUsername: "admin", + wantDatabase: "testdb", + }, + { + name: "postgresql with query params for credentials", + data: `jdbc:postgresql://localhost:1521/testdb?sslmode=disable&password=testpassword&user=testuser`, + wantHost: "localhost:1521", + wantUsername: "testuser", + wantDatabase: "testdb", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + if err != nil { + t.Fatalf("FromData() error = %v", err) + } + if len(results) == 0 { + t.Fatal("expected at least one result") + } + r := results[0] + if got := r.ExtraData["host"]; got != tt.wantHost { + t.Errorf("ExtraData[host] = %q, want %q", got, tt.wantHost) + } + if got := r.ExtraData["username"]; got != tt.wantUsername { + t.Errorf("ExtraData[username] = %q, want %q", got, tt.wantUsername) + } + if got := r.ExtraData["database"]; got != tt.wantDatabase { + t.Errorf("ExtraData[database] = %q, want %q", got, tt.wantDatabase) + } + }) + } +} + +func TestJdbc_ExtraData_UnsupportedSubprotocol(t *testing.T) { + // For unsupported subprotocols (e.g., sqlite), ExtraData should be nil + // because we can't parse connection info, but the result should still be returned. + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(`jdbc:sqlite:/data/test.db`)) + if err != nil { + t.Fatalf("FromData() error = %v", err) + } + if len(results) == 0 { + t.Fatal("expected at least one result") + } + r := results[0] + if r.ExtraData != nil { + t.Errorf("expected nil ExtraData for unsupported subprotocol, got %v", r.ExtraData) + } +} + func TestJdbc_FromDataWithIgnorePattern(t *testing.T) { type args struct { ctx context.Context diff --git a/pkg/detectors/mongodb/mongodb.go b/pkg/detectors/mongodb/mongodb.go index df507d75f2a0..957f1140d913 100644 --- a/pkg/detectors/mongodb/mongodb.go +++ b/pkg/detectors/mongodb/mongodb.go @@ -45,7 +45,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result logger := logContext.AddLogger(ctx).Logger().WithName("mongodb") dataStr := string(data) - uniqueMatches := make(map[string]string) + type mongoMatch struct { + password string + parsedURL *url.URL + } + uniqueMatches := make(map[string]mongoMatch) for _, match := range connStrPat.FindAllStringSubmatch(dataStr, -1) { // Filter out common placeholder passwords. password := match[3] @@ -78,16 +82,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result connUrl.RawQuery = params.Encode() connStr = connUrl.String() - uniqueMatches[connStr] = password + uniqueMatches[connStr] = mongoMatch{password: password, parsedURL: connUrl} } - for connStr, password := range uniqueMatches { + for connStr, m := range uniqueMatches { + extraData := map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", + } + if m.parsedURL.Host != "" { + extraData["host"] = m.parsedURL.Host + } + if m.parsedURL.User != nil && m.parsedURL.User.Username() != "" { + extraData["username"] = m.parsedURL.User.Username() + } + if db := strings.TrimPrefix(m.parsedURL.Path, "/"); db != "" { + extraData["database"] = db + } + r := detectors.Result{ DetectorType: detectorspb.DetectorType_MongoDB, Raw: []byte(connStr), - ExtraData: map[string]string{ - "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", - }, + ExtraData: extraData, } if verify { @@ -101,7 +116,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if isErrDeterminate(vErr) { continue } - r.SetVerificationError(vErr, password) + r.SetVerificationError(vErr, m.password) if isVerified { r.AnalysisInfo = map[string]string{ diff --git a/pkg/detectors/mongodb/mongodb_test.go b/pkg/detectors/mongodb/mongodb_test.go index 9482fcca7cf3..7801a71efcc3 100644 --- a/pkg/detectors/mongodb/mongodb_test.go +++ b/pkg/detectors/mongodb/mongodb_test.go @@ -5,6 +5,87 @@ import ( "testing" ) +func TestMongoDB_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + wantDatabase string + }{ + { + name: "single host with port", + data: `mongodb://myDBReader:D1fficultP%40ssw0rd@mongodb0.example.com:27017`, + wantHost: "mongodb0.example.com:27017", + wantUsername: "myDBReader", + }, + { + name: "single host without port", + data: `mongodb://myDBReader:D1fficultP%40ssw0rd@mongodb0.example.com`, + wantHost: "mongodb0.example.com", + wantUsername: "myDBReader", + }, + { + name: "with options and no database", + data: `mongodb://username:password@host.docker.internal:27018/?authMechanism=PLAIN&tls=true`, + wantHost: "host.docker.internal:27018", + wantUsername: "username", + }, + { + name: "cosmos db style with database", + data: `mongodb://agenda-live:m21w7PFfRXQwfHZU1Fgx0rTX29ZBQaWMODLeAjsmyslVcMmcmy6CnLyu3byVDtdLYcCokze8lIE4KyAgSCGZxQ==@agenda-live.mongo.cosmos.azure.com:10255/csb-db?retryWrites=false&ssl=true&replicaSet=globaldb&maxIdleTimeMS=120000&appName=@agenda-live@`, + wantHost: "agenda-live.mongo.cosmos.azure.com:10255", + wantUsername: "agenda-live", + wantDatabase: "csb-db", + }, + { + name: "with database in path", + data: `mongodb://db-user:db-password@mongodb-instance:27017/db-name`, + wantHost: "mongodb-instance:27017", + wantUsername: "db-user", + wantDatabase: "db-name", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + if err != nil { + t.Fatalf("FromData() error = %v", err) + } + if len(results) == 0 { + t.Fatal("expected at least one result") + } + r := results[0] + if got := r.ExtraData["host"]; got != tt.wantHost { + t.Errorf("ExtraData[host] = %q, want %q", got, tt.wantHost) + } + if tt.wantUsername != "" { + if got := r.ExtraData["username"]; got != tt.wantUsername { + t.Errorf("ExtraData[username] = %q, want %q", got, tt.wantUsername) + } + } else { + if got, ok := r.ExtraData["username"]; ok { + t.Errorf("ExtraData[username] should be absent, got %q", got) + } + } + if tt.wantDatabase != "" { + if got := r.ExtraData["database"]; got != tt.wantDatabase { + t.Errorf("ExtraData[database] = %q, want %q", got, tt.wantDatabase) + } + } else { + if got, ok := r.ExtraData["database"]; ok { + t.Errorf("ExtraData[database] should be absent, got %q", got) + } + } + if got := r.ExtraData["rotation_guide"]; got == "" { + t.Error("ExtraData[rotation_guide] should still be present") + } + }) + } +} + func TestMongoDB_Pattern(t *testing.T) { tests := []struct { name string diff --git a/pkg/detectors/postgres/postgres.go b/pkg/detectors/postgres/postgres.go index 3a7449e052f5..014dfa5b33b6 100644 --- a/pkg/detectors/postgres/postgres.go +++ b/pkg/detectors/postgres/postgres.go @@ -177,6 +177,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete result.ExtraData = map[string]string{ pgSslmode: sslmode, } + if host != "" { + if port != "" { + result.ExtraData["host"] = host + ":" + port + } else { + result.ExtraData["host"] = host + } + } + if user != "" { + result.ExtraData["username"] = user + } + if dbname := params[pgDbname]; dbname != "" { + result.ExtraData["database"] = dbname + } results = append(results, result) } diff --git a/pkg/detectors/postgres/postgres_test.go b/pkg/detectors/postgres/postgres_test.go index 85f5b82eb507..bd6983f29554 100644 --- a/pkg/detectors/postgres/postgres_test.go +++ b/pkg/detectors/postgres/postgres_test.go @@ -82,6 +82,63 @@ func TestPostgres_Pattern(t *testing.T) { } } +func TestPostgres_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + wantDatabase string + }{ + { + name: "standard URI with database", + data: "postgres://myuser:mypass@dbhost.example.com:5432/mydb", + wantHost: "dbhost.example.com:5432", + wantUsername: "myuser", + wantDatabase: "mydb", + }, + { + name: "postgresql scheme", + data: "postgresql://admin:secret@10.0.0.1:5433/production", + wantHost: "10.0.0.1:5433", + wantUsername: "admin", + wantDatabase: "production", + }, + { + name: "without database", + data: "postgres://sN19x:d7N8bs@1.2.3.4:5432?sslmode=require", + wantHost: "1.2.3.4:5432", + wantUsername: "sN19x", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{detectLoopback: true} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + if err != nil { + t.Fatalf("FromData() error = %v", err) + } + if len(results) == 0 { + t.Fatal("expected at least one result") + } + r := results[0] + if got := r.ExtraData["host"]; got != tt.wantHost { + t.Errorf("ExtraData[host] = %q, want %q", got, tt.wantHost) + } + if got := r.ExtraData["username"]; got != tt.wantUsername { + t.Errorf("ExtraData[username] = %q, want %q", got, tt.wantUsername) + } + if got := r.ExtraData["database"]; got != tt.wantDatabase { + t.Errorf("ExtraData[database] = %q, want %q", got, tt.wantDatabase) + } + if _, ok := r.ExtraData["sslmode"]; !ok { + t.Error("ExtraData[sslmode] should still be present") + } + }) + } +} + func TestPostgres_FromDataWithIgnorePattern(t *testing.T) { s := New( WithIgnorePattern([]string{ diff --git a/pkg/detectors/redis/redis.go b/pkg/detectors/redis/redis.go index 633c9834c157..3432624ead24 100644 --- a/pkg/detectors/redis/redis.go +++ b/pkg/detectors/redis/redis.go @@ -62,6 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detectorspb.DetectorType_Redis, Raw: []byte(urlMatch), Redacted: redact, + ExtraData: extraDataFromURL(parsedURL), } if verify { @@ -101,6 +102,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detectorspb.DetectorType_Redis, Raw: []byte(urlMatch), Redacted: redact, + ExtraData: extraDataFromURL(parsedURL), } if verify { @@ -120,6 +122,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func extraDataFromURL(u *url.URL) map[string]string { + extraData := make(map[string]string) + if u.Host != "" { + extraData["host"] = u.Host + } + if u.User != nil && u.User.Username() != "" { + extraData["username"] = u.User.Username() + } + return extraData +} + func verifyRedis(ctx context.Context, u *url.URL) bool { opt, err := redis.ParseURL(u.String()) if err != nil { diff --git a/pkg/detectors/redis/redis_test.go b/pkg/detectors/redis/redis_test.go index fccd171ab2c2..55573d5eb95f 100644 --- a/pkg/detectors/redis/redis_test.go +++ b/pkg/detectors/redis/redis_test.go @@ -22,6 +22,59 @@ var ( keyword = "redis" ) +func TestRedis_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + }{ + { + name: "standard redis URI", + data: `redis://myuser:mysecretpass@redis.example.com:6379/0`, + wantHost: "redis.example.com:6379", + wantUsername: "myuser", + }, + { + name: "redis URI with default username", + data: `redis://default:mysecretpass@redis.example.com:6379`, + wantHost: "redis.example.com:6379", + wantUsername: "default", + }, + { + name: "azure redis pattern without username", + data: `mycache.redis.cache.windows.net:6380,password=Xcc3S9d7And6aMdfOcUc0acHJh3CiDh3l9DsapNwGwyS,ssl=True,abortConnect=False`, + wantHost: "mycache.redis.cache.windows.net:6380", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + if err != nil { + t.Fatalf("FromData() error = %v", err) + } + if len(results) == 0 { + t.Fatal("expected at least one result") + } + r := results[0] + if got := r.ExtraData["host"]; got != tt.wantHost { + t.Errorf("ExtraData[host] = %q, want %q", got, tt.wantHost) + } + if tt.wantUsername != "" { + if got := r.ExtraData["username"]; got != tt.wantUsername { + t.Errorf("ExtraData[username] = %q, want %q", got, tt.wantUsername) + } + } else { + if got, ok := r.ExtraData["username"]; ok { + t.Errorf("ExtraData[username] should be absent, got %q", got) + } + } + }) + } +} + func TestRedisIntegration_Pattern(t *testing.T) { d := Scanner{} ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})