| 676 | |
| 677 | |
| 678 | FileEncoding ASConsole::readFile(const string& fileName_, stringstream& in) const |
| 679 | { |
| 680 | const int blockSize = 131072; // 128 KB |
| 681 | ifstream fin(fileName_.c_str(), ios::binary); |
| 682 | if (!fin) |
| 683 | error("Cannot open input file", fileName_.c_str()); |
| 684 | char data[blockSize]; |
| 685 | fin.read(data, sizeof(data)); |
| 686 | if (fin.bad()) |
| 687 | error("Cannot read input file", fileName_.c_str()); |
| 688 | size_t dataSize = static_cast<size_t>(fin.gcount()); |
| 689 | FileEncoding encoding = detectEncoding(data, dataSize); |
| 690 | if (encoding == UTF_32BE || encoding == UTF_32LE) |
| 691 | error(_("Cannot process UTF-32 encoding"), fileName_.c_str()); |
| 692 | bool firstBlock = true; |
| 693 | while (dataSize) |
| 694 | { |
| 695 | if (encoding == UTF_16LE || encoding == UTF_16BE) |
| 696 | { |
| 697 | // convert utf-16 to utf-8 |
| 698 | size_t utf8Size = Utf8Length(data, dataSize, encoding); |
| 699 | char* utf8Out = new char[utf8Size]; |
| 700 | size_t utf8Len = Utf16ToUtf8(data, dataSize, encoding, firstBlock, utf8Out); |
| 701 | assert(utf8Len == utf8Size); |
| 702 | in << string(utf8Out, utf8Len); |
| 703 | delete []utf8Out; |
| 704 | } |
| 705 | else |
| 706 | in << string(data, dataSize); |
| 707 | fin.read(data, sizeof(data)); |
| 708 | if (fin.bad()) |
| 709 | error("Cannot read input file", fileName_.c_str()); |
| 710 | dataSize = static_cast<size_t>(fin.gcount()); |
| 711 | firstBlock = false; |
| 712 | } |
| 713 | fin.close(); |
| 714 | return encoding; |
| 715 | } |
| 716 | |
| 717 | void ASConsole::setIgnoreExcludeErrors(bool state) |
| 718 | { ignoreExcludeErrors = state; } |