TestUTF8Tokenization tests tokenization with UTF-8 characters
(t *testing.T)
| 425 | |
| 426 | // TestUTF8Tokenization tests tokenization with UTF-8 characters |
| 427 | func TestUTF8Tokenization(t *testing.T) { |
| 428 | // Setup |
| 429 | key := []byte("0123456789ABCDEF0123456789ABCDEF") // 32-byte key for AES-256 |
| 430 | service := NewTokenizationService(key) |
| 431 | |
| 432 | // Test cases with UTF-8 characters mentioned in the bug report |
| 433 | testData := []string{ |
| 434 | "Taizé", // é character that caused the original bug |
| 435 | "José", // Common Spanish name with accent |
| 436 | "François", // French name with ç |
| 437 | "Müller", // German name with umlaut ü |
| 438 | "Björk", // Scandinavian name with ö |
| 439 | "Tschüß", // German word with ü and ß |
| 440 | "Ångström", // Swedish name with Å |
| 441 | "Zoë", // Name with ë |
| 442 | "Pañuelos", // Spanish word with ñ |
| 443 | "Café", // Common word with é |
| 444 | "naïve", // Word with ï |
| 445 | "résumé", // Word with multiple accents |
| 446 | "José María Aznar", // Full name with spaces and accents |
| 447 | "北京", // Chinese characters |
| 448 | "東京", // Japanese characters |
| 449 | "москва", // Russian characters |
| 450 | "العربية", // Arabic characters |
| 451 | "हिन्दी", // Hindi characters |
| 452 | "🔒🔑", // Emoji (already tested but keeping for UTF-8 completeness) |
| 453 | } |
| 454 | |
| 455 | for _, original := range testData { |
| 456 | t.Run(fmt.Sprintf("Standard_%s", original), func(t *testing.T) { |
| 457 | // Test standard tokenization |
| 458 | token, err := service.Tokenize(original) |
| 459 | if err != nil { |
| 460 | t.Errorf("Tokenize(%q) error: %v", original, err) |
| 461 | return |
| 462 | } |
| 463 | |
| 464 | // Verify token is valid base64 |
| 465 | _, err = base64.StdEncoding.DecodeString(token) |
| 466 | if err != nil { |
| 467 | t.Errorf("Token %q is not valid base64", token) |
| 468 | return |
| 469 | } |
| 470 | |
| 471 | // Test detokenization |
| 472 | decrypted, err := service.Detokenize(token) |
| 473 | if err != nil { |
| 474 | t.Errorf("Detokenize(%q) error: %v", token, err) |
| 475 | return |
| 476 | } |
| 477 | |
| 478 | // Verify the result matches exactly |
| 479 | if decrypted != original { |
| 480 | t.Errorf("Expected %q, got %q", original, decrypted) |
| 481 | } |
| 482 | }) |
| 483 | |
| 484 | t.Run(fmt.Sprintf("FormatPreserving_%s", original), func(t *testing.T) { |
nothing calls this directly
no test coverage detected