| 8 | using namespace std; |
| 9 | |
| 10 | int main(int argc, char** argv) |
| 11 | { |
| 12 | if (argc != 2) { |
| 13 | cout << "\nUsage: docsample filename\n"; |
| 14 | return 0; |
| 15 | } |
| 16 | const char* test_file_path = argv[1]; |
| 17 | // Open the test file (must be UTF-8 encoded) |
| 18 | ifstream fs8(test_file_path); |
| 19 | if (!fs8.is_open()) { |
| 20 | cout << "Could not open " << test_file_path << endl; |
| 21 | return 0; |
| 22 | } |
| 23 | |
| 24 | unsigned line_count = 1; |
| 25 | string line; |
| 26 | // Play with all the lines in the file |
| 27 | while (getline(fs8, line)) { |
| 28 | // check for invalid utf-8 (for a simple yes/no check, there is also utf8::is_valid function) |
| 29 | #if __cplusplus >= 201103L // C++ 11 or later |
| 30 | auto end_it = utf8::find_invalid(line.begin(), line.end()); |
| 31 | #else |
| 32 | string::iterator end_it = utf8::find_invalid(line.begin(), line.end()); |
| 33 | #endif // C++ 11 |
| 34 | if (end_it != line.end()) { |
| 35 | cout << "Invalid UTF-8 encoding detected at line " << line_count << "\n"; |
| 36 | cout << "This part is fine: " << string(line.begin(), end_it) << "\n"; |
| 37 | } |
| 38 | // Get the line length (at least for the valid part) |
| 39 | ptrdiff_t length = utf8::distance(line.begin(), end_it); |
| 40 | cout << "Length of line " << line_count << " is " << length << "\n"; |
| 41 | |
| 42 | // Convert it to utf-16 |
| 43 | #if __cplusplus >= 201103L // C++ 11 or later |
| 44 | u16string utf16line = utf8::utf8to16(line); |
| 45 | #else |
| 46 | vector<unsigned short> utf16line; |
| 47 | utf8::utf8to16(line.begin(), end_it, back_inserter(utf16line)); |
| 48 | #endif // C++ 11 |
| 49 | // And back to utf-8; |
| 50 | #if __cplusplus >= 201103L // C++ 11 or later |
| 51 | string utf8line = utf8::utf16to8(utf16line); |
| 52 | #else |
| 53 | string utf8line; |
| 54 | utf8::utf16to8(utf16line.begin(), utf16line.end(), back_inserter(utf8line)); |
| 55 | #endif // C++ 11 |
| 56 | // Confirm that the conversion went OK: |
| 57 | if (utf8line != string(line.begin(), end_it)) |
| 58 | cout << "Error in UTF-16 conversion at line: " << line_count << "\n"; |
| 59 | |
| 60 | line_count++; |
| 61 | } |
| 62 | |
| 63 | return 0; |
| 64 | } |