TestTokenization reads the *.txt and *.tokenizer.json files from the ../testdata directory, then runs the tokenizer.Tokenize function on the text input and compares the output to the events read from the JSON files.
(t *testing.T)
| 17 | // TestTokenization reads the *.txt and *.tokenizer.json files from the ../testdata directory, then runs |
| 18 | // the tokenizer.Tokenize function on the text input and compares the output to the events read from the JSON files. |
| 19 | func TestTokenization(t *testing.T) { |
| 20 | t.Logf("Locating testdata directory...") |
| 21 | tryDirectories := []string{ |
| 22 | "./testdata", |
| 23 | "../testdata", |
| 24 | } |
| 25 | |
| 26 | foundDir := "" |
| 27 | for _, dir := range tryDirectories { |
| 28 | if _, err := os.Stat(dir); err == nil { |
| 29 | foundDir = dir |
| 30 | } |
| 31 | } |
| 32 | if foundDir == "" { |
| 33 | t.Fatalf("failed to find testdata directory in %v", tryDirectories) |
| 34 | } |
| 35 | t.Logf("Testdata directory is located at %s.", foundDir) |
| 36 | |
| 37 | if e := filepath.Walk(foundDir, func(path string, info fs.FileInfo, _ error) error { |
| 38 | if info.IsDir() { |
| 39 | return nil |
| 40 | } |
| 41 | |
| 42 | if !strings.HasSuffix(path, ".txt") { |
| 43 | return nil |
| 44 | } |
| 45 | base := strings.Replace(path2.Base(filepath.ToSlash(path)), ".txt", "", 1) |
| 46 | t.Run( |
| 47 | base, |
| 48 | func(t *testing.T) { |
| 49 | t.Parallel() |
| 50 | sourceFile := path |
| 51 | expectedFile := strings.Replace(path, ".txt", ".tokenizer.json", 1) |
| 52 | |
| 53 | t.Logf("Tokenizing %s and comparing with %s...", sourceFile, expectedFile) |
| 54 | |
| 55 | var source, err = os.ReadFile(sourceFile) |
| 56 | if err != nil { |
| 57 | t.Fatalf("failed to read source file %s (%v)", source, err) |
| 58 | } |
| 59 | reader := tokenizer.Tokenize(bytes.NewReader(source)) |
| 60 | var events []tokenizer.Event |
| 61 | for { |
| 62 | event, ok := <-reader |
| 63 | if !ok { |
| 64 | break |
| 65 | } |
| 66 | events = append(events, event) |
| 67 | } |
| 68 | expectedFh, err := os.Open(expectedFile) |
| 69 | if err != nil { |
| 70 | t.Logf( |
| 71 | "Failed to read expected file %s, writing actual output to %s (%v).", |
| 72 | expectedFile, |
| 73 | expectedFile+".actual", |
| 74 | err, |
| 75 | ) |
| 76 | expectedFh, err = os.Create(expectedFile + ".actual") |