자막 테스트 중
This commit is contained in:
parent
3205a29fa6
commit
5e5619e8d8
29 changed files with 165728 additions and 4 deletions
|
|
@ -1,8 +1,97 @@
|
||||||
import 'package:dart_framework/core/application.dart';
|
import 'dart:convert';
|
||||||
|
import 'dart:io';
|
||||||
import 'package:dart_framework/log/log.dart';
|
import 'package:dart_framework/log/log.dart';
|
||||||
|
import 'package:dart_framework/subtitle/ParserSRT.dart';
|
||||||
|
import 'package:path/path.dart' as path;
|
||||||
|
|
||||||
|
import 'package:dart_framework/core/application.dart';
|
||||||
|
import 'package:dart_framework/charset.dart';
|
||||||
|
|
||||||
void main() async {
|
void main() async {
|
||||||
Application(() {
|
Application(() async {
|
||||||
test();
|
var filePath = Platform.script.toFilePath(windows: true);
|
||||||
}, (error, stack) {});
|
var file = File('${path.dirname(path.dirname(filePath))}/test_euckr.smi');
|
||||||
|
var bytes = file.readAsBytesSync();
|
||||||
|
var encoding = Charset.detect(bytes, defaultEncoding: Utf8Codec());
|
||||||
|
print(encoding);
|
||||||
|
var result = eucKr.decode(bytes);
|
||||||
|
// print(result);
|
||||||
|
|
||||||
|
//SRT
|
||||||
|
// var parser = ParserSRT();
|
||||||
|
// var subtitles = parser.getSubtitlesData(result!, SubtitleType.srt);
|
||||||
|
// var index = 0;
|
||||||
|
// for (var item in subtitles.subtitles) {
|
||||||
|
// print('$index. ${item.text}');
|
||||||
|
// index++;
|
||||||
|
// }
|
||||||
|
|
||||||
|
//SMI
|
||||||
|
var smi = ParserSMI();
|
||||||
|
smi.getSubtitlesData(result!);
|
||||||
|
|
||||||
|
// var regExp = RegExp(
|
||||||
|
// r'<SYNC Start=(\d+)><P Class=KRCC>',
|
||||||
|
// caseSensitive: false,
|
||||||
|
// multiLine: true,
|
||||||
|
// );
|
||||||
|
}, (error, stack) {
|
||||||
|
print(error);
|
||||||
|
print(stack);
|
||||||
|
exit(0);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
class ParserSMI {
|
||||||
|
Subtitles getSubtitlesData(String subtitlesContent) {
|
||||||
|
subtitlesContent = subtitlesContent.replaceAll(
|
||||||
|
RegExp('<font (.*?=)(.*?>)|</font>|<b>|</b>| |<i>|</i>',
|
||||||
|
caseSensitive: false, multiLine: false),
|
||||||
|
'');
|
||||||
|
|
||||||
|
var regExp = RegExp(
|
||||||
|
r'<SYNC Start=(\d+)><P Class=(.*?>)(\r|\n)*((((?!<SYNC|<\/BODY).*)(\r|\n))+)',
|
||||||
|
caseSensitive: false,
|
||||||
|
multiLine: true,
|
||||||
|
);
|
||||||
|
final matches = regExp.allMatches(subtitlesContent).toList();
|
||||||
|
final subtitleList = <Subtitle>[];
|
||||||
|
Subtitle? lastItem;
|
||||||
|
for (final regExpMatch in matches) {
|
||||||
|
var startTime = Time(
|
||||||
|
Duration(milliseconds: int.parse(regExpMatch.group(1).toString())));
|
||||||
|
var content = regExpMatch.group(4)!;
|
||||||
|
if (lastItem != null) {
|
||||||
|
lastItem.endTime.duration = startTime.before;
|
||||||
|
lastItem = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!content.contains(' ')) {
|
||||||
|
content = content
|
||||||
|
.replaceAll(RegExp(r'(\r\n|\r|\n)', caseSensitive: false), '')
|
||||||
|
.replaceAll(RegExp(r'(<br>|<\/br>)', caseSensitive: false), ' ')
|
||||||
|
.trim();
|
||||||
|
|
||||||
|
if (content.isNotEmpty) {
|
||||||
|
if (subtitleList.isNotEmpty &&
|
||||||
|
content.contains(subtitleList.last.text)) {
|
||||||
|
subtitleList.last.text = content;
|
||||||
|
lastItem = subtitleList.last;
|
||||||
|
} else {
|
||||||
|
var endTime = Time(Duration());
|
||||||
|
lastItem =
|
||||||
|
Subtitle(startTime: startTime, endTime: endTime, text: content);
|
||||||
|
subtitleList.add(lastItem);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (var item in subtitleList) {
|
||||||
|
print(
|
||||||
|
'${item.startTime.toString()} --> ${item.endTime.toString()} >> ${item.text}');
|
||||||
|
}
|
||||||
|
|
||||||
|
return Subtitles(subtitles: subtitleList);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
12
lib/charset.dart
Normal file
12
lib/charset.dart
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
library charset;
|
||||||
|
|
||||||
|
export 'charset/charset.dart';
|
||||||
|
|
||||||
|
export 'charset/code_page.dart';
|
||||||
|
export 'charset/euc_jp.dart';
|
||||||
|
export 'charset/euc_kr.dart';
|
||||||
|
export 'charset/gbk.dart';
|
||||||
|
export 'charset/shift_jis.dart';
|
||||||
|
|
||||||
|
export 'charset/utf/utf16.dart';
|
||||||
|
export 'charset/utf/utf32.dart';
|
||||||
525
lib/charset/charset.dart
Normal file
525
lib/charset/charset.dart
Normal file
|
|
@ -0,0 +1,525 @@
|
||||||
|
import 'dart:convert';
|
||||||
|
|
||||||
|
import 'code_page.dart';
|
||||||
|
import 'euc_jp.dart';
|
||||||
|
import 'euc_kr.dart';
|
||||||
|
import 'gbk.dart';
|
||||||
|
import 'shift_jis.dart';
|
||||||
|
import 'utf/utf16.dart';
|
||||||
|
import 'utf/utf32.dart';
|
||||||
|
|
||||||
|
/// Global method for Encoders
|
||||||
|
class Charset {
|
||||||
|
/// Encoding map
|
||||||
|
static final _charCodeMap = <String, Encoding>{
|
||||||
|
'iso-ir-101': latin2,
|
||||||
|
'iso_8859-2': latin2,
|
||||||
|
'iso-8859-2': latin2,
|
||||||
|
'latin2': latin2,
|
||||||
|
'l2': latin2,
|
||||||
|
'csisolatin2': latin2,
|
||||||
|
|
||||||
|
'iso-ir-109': latin3,
|
||||||
|
'iso_8859-3': latin3,
|
||||||
|
'iso-8859-3': latin3,
|
||||||
|
'latin3': latin3,
|
||||||
|
'l3': latin3,
|
||||||
|
'csisolatin3': latin3,
|
||||||
|
|
||||||
|
'iso-ir-110': latin4,
|
||||||
|
'iso_8859-4': latin4,
|
||||||
|
'iso-8859-4': latin4,
|
||||||
|
'latin4': latin4,
|
||||||
|
'l4': latin4,
|
||||||
|
'csisolatin4': latin4,
|
||||||
|
|
||||||
|
'iso-ir-144': latinCyrillic,
|
||||||
|
'iso_8859-5': latinCyrillic,
|
||||||
|
'iso-8859-5': latinCyrillic,
|
||||||
|
'cyrillic': latinCyrillic,
|
||||||
|
'csisolatincyrillic': latinCyrillic,
|
||||||
|
|
||||||
|
'iso-ir-127': latinArabic,
|
||||||
|
'iso_8859-6': latinArabic,
|
||||||
|
'iso-8859-6': latinArabic,
|
||||||
|
'ecma-114': latinArabic,
|
||||||
|
'asmo-708': latinArabic,
|
||||||
|
'arabic': latinArabic,
|
||||||
|
'csisolatinarabic': latinArabic,
|
||||||
|
|
||||||
|
'iso-ir-126': latinGreek,
|
||||||
|
'iso_8859-7': latinGreek,
|
||||||
|
'iso-8859-7': latinGreek,
|
||||||
|
'elot_928': latinGreek,
|
||||||
|
'ecma-118': latinGreek,
|
||||||
|
'greek': latinGreek,
|
||||||
|
'greek8': latinGreek,
|
||||||
|
'csisolatingreek': latinGreek,
|
||||||
|
|
||||||
|
'iso-ir-138': latinHebrew,
|
||||||
|
'iso_8859-8': latinHebrew,
|
||||||
|
'iso-8859-8': latinHebrew,
|
||||||
|
'hebrew': latinHebrew,
|
||||||
|
'csisolatinhebrew': latinHebrew,
|
||||||
|
|
||||||
|
'iso-ir-148': latin5,
|
||||||
|
'iso_8859-9': latin5,
|
||||||
|
'iso-8859-9': latin5,
|
||||||
|
'latin5': latin5,
|
||||||
|
'l5': latin5,
|
||||||
|
'csisolatin5': latin5,
|
||||||
|
|
||||||
|
'iso-ir-157': latin6,
|
||||||
|
'l6': latin6,
|
||||||
|
'iso-8859-10': latin6,
|
||||||
|
'iso_8859-10': latin6,
|
||||||
|
'iso_8859-10:1992': latin6,
|
||||||
|
'csisolatin6': latin6,
|
||||||
|
'latin6': latin6,
|
||||||
|
|
||||||
|
'tis-620': latinThai,
|
||||||
|
'cstis620': latinThai,
|
||||||
|
'iso-8859-11': latinThai,
|
||||||
|
|
||||||
|
'iso-8859-13': latin7,
|
||||||
|
'csiso885913': latin7,
|
||||||
|
|
||||||
|
'iso-8859-14': latin8,
|
||||||
|
'iso-ir-199': latin8,
|
||||||
|
'iso_8859-14:1998': latin8,
|
||||||
|
'iso_8859-14': latin8,
|
||||||
|
'latin8': latin8,
|
||||||
|
'iso-celtic': latin8,
|
||||||
|
'l8': latin8,
|
||||||
|
'csiso885914': latin8,
|
||||||
|
|
||||||
|
'iso-8859-15': latin9,
|
||||||
|
'iso_8859-15': latin9,
|
||||||
|
'latin-9': latin9,
|
||||||
|
'csiso885915': latin9,
|
||||||
|
|
||||||
|
'iso-8859-16': latin10,
|
||||||
|
'iso-ir-226': latin10,
|
||||||
|
'iso_8859-16:2001': latin10,
|
||||||
|
'iso_8859-16': latin10,
|
||||||
|
'latin10': latin10,
|
||||||
|
'l10': latin10,
|
||||||
|
'csiso885916': latin10,
|
||||||
|
|
||||||
|
// cp437
|
||||||
|
'437': cp437,
|
||||||
|
'cp437': cp437,
|
||||||
|
'cp-437': cp437,
|
||||||
|
'ibm437': cp437,
|
||||||
|
'csibm437': cp437,
|
||||||
|
'cspc8codepage437': cp437,
|
||||||
|
|
||||||
|
'737': cp737,
|
||||||
|
'cp737': cp737,
|
||||||
|
'cp-737': cp737,
|
||||||
|
'ibm737': cp737,
|
||||||
|
'csibm737': cp737,
|
||||||
|
|
||||||
|
'775': cp775,
|
||||||
|
'cp775': cp775,
|
||||||
|
'cp-775': cp775,
|
||||||
|
'ibm775': cp775,
|
||||||
|
'csibm775': cp775,
|
||||||
|
|
||||||
|
'850': cp850,
|
||||||
|
'cp850': cp850,
|
||||||
|
'cp-850': cp850,
|
||||||
|
'ibm850': cp850,
|
||||||
|
'csibm850': cp850,
|
||||||
|
|
||||||
|
'852': cp852,
|
||||||
|
'cp852': cp852,
|
||||||
|
'cp-852': cp852,
|
||||||
|
'ibm852': cp852,
|
||||||
|
'csibm852': cp852,
|
||||||
|
|
||||||
|
'855': cp855,
|
||||||
|
'cp855': cp855,
|
||||||
|
'cp-855': cp855,
|
||||||
|
'ibm855': cp855,
|
||||||
|
'csibm855': cp855,
|
||||||
|
|
||||||
|
'856': cp856,
|
||||||
|
'cp856': cp856,
|
||||||
|
'cp-856': cp856,
|
||||||
|
'ibm856': cp856,
|
||||||
|
'csibm856': cp856,
|
||||||
|
|
||||||
|
'857': cp857,
|
||||||
|
'cp857': cp857,
|
||||||
|
'cp-857': cp857,
|
||||||
|
'ibm857': cp857,
|
||||||
|
'csibm857': cp857,
|
||||||
|
|
||||||
|
'858': cp858,
|
||||||
|
'cp858': cp858,
|
||||||
|
'cp-858': cp858,
|
||||||
|
'ibm858': cp858,
|
||||||
|
'csibm858': cp858,
|
||||||
|
|
||||||
|
'860': cp860,
|
||||||
|
'cp860': cp860,
|
||||||
|
'cp-860': cp860,
|
||||||
|
'ibm860': cp860,
|
||||||
|
'csibm860': cp860,
|
||||||
|
|
||||||
|
'861': cp861,
|
||||||
|
'cp861': cp861,
|
||||||
|
'cp-861': cp861,
|
||||||
|
'ibm861': cp861,
|
||||||
|
'csibm861': cp861,
|
||||||
|
|
||||||
|
'862': cp862,
|
||||||
|
'cp862': cp862,
|
||||||
|
'cp-862': cp862,
|
||||||
|
'ibm862': cp862,
|
||||||
|
'csibm862': cp862,
|
||||||
|
|
||||||
|
'863': cp863,
|
||||||
|
'cp863': cp863,
|
||||||
|
'cp-863': cp863,
|
||||||
|
'ibm863': cp863,
|
||||||
|
'csibm863': cp863,
|
||||||
|
|
||||||
|
'864': cp864,
|
||||||
|
'cp864': cp864,
|
||||||
|
'cp-864': cp864,
|
||||||
|
'ibm864': cp864,
|
||||||
|
'csibm864': cp864,
|
||||||
|
|
||||||
|
'865': cp865,
|
||||||
|
'cp865': cp865,
|
||||||
|
'cp-865': cp865,
|
||||||
|
'ibm865': cp865,
|
||||||
|
'csibm865': cp865,
|
||||||
|
|
||||||
|
'866': cp866,
|
||||||
|
'cp866': cp866,
|
||||||
|
'cp-866': cp866,
|
||||||
|
'ibm866': cp866,
|
||||||
|
'csibm866': cp866,
|
||||||
|
|
||||||
|
'869': cp869,
|
||||||
|
'cp869': cp869,
|
||||||
|
'cp-869': cp869,
|
||||||
|
'ibm869': cp869,
|
||||||
|
'csibm869': cp869,
|
||||||
|
|
||||||
|
'922': cp922,
|
||||||
|
'cp922': cp922,
|
||||||
|
'cp-922': cp922,
|
||||||
|
'ibm922': cp922,
|
||||||
|
'csibm922': cp922,
|
||||||
|
|
||||||
|
'1046': cp1046,
|
||||||
|
'cp1046': cp1046,
|
||||||
|
'cp-1046': cp1046,
|
||||||
|
'ibm1046': cp1046,
|
||||||
|
'csibm1046': cp1046,
|
||||||
|
|
||||||
|
'1124': cp1124,
|
||||||
|
'cp1124': cp1124,
|
||||||
|
'cp-1124': cp1124,
|
||||||
|
'ibm1124': cp1124,
|
||||||
|
'csibm1124': cp1124,
|
||||||
|
|
||||||
|
'1125': cp1125,
|
||||||
|
'cp1125': cp1125,
|
||||||
|
'cp-1125': cp1125,
|
||||||
|
'ibm1125': cp1125,
|
||||||
|
'csibm1125': cp1125,
|
||||||
|
|
||||||
|
'1129': cp1129,
|
||||||
|
'cp1129': cp1129,
|
||||||
|
'cp-1129': cp1129,
|
||||||
|
'ibm1129': cp1129,
|
||||||
|
'csibm1129': cp1129,
|
||||||
|
|
||||||
|
'1133': cp1133,
|
||||||
|
'cp1133': cp1133,
|
||||||
|
'cp-1133': cp1133,
|
||||||
|
'ibm1133': cp1133,
|
||||||
|
'csibm1133': cp1133,
|
||||||
|
|
||||||
|
'1161': cp1161,
|
||||||
|
'cp1161': cp1161,
|
||||||
|
'cp-1161': cp1161,
|
||||||
|
'ibm1161': cp1161,
|
||||||
|
'csibm1161': cp1161,
|
||||||
|
|
||||||
|
'1162': cp1162,
|
||||||
|
'cp1162': cp1162,
|
||||||
|
'cp-1162': cp1162,
|
||||||
|
'ibm1162': cp1162,
|
||||||
|
'csibm1162': cp1162,
|
||||||
|
|
||||||
|
'1163': cp1163,
|
||||||
|
'cp1163': cp1163,
|
||||||
|
'cp-1163': cp1163,
|
||||||
|
'ibm1163': cp1163,
|
||||||
|
'csibm1163': cp1163,
|
||||||
|
|
||||||
|
'874': windows874,
|
||||||
|
'cp874': windows874,
|
||||||
|
'cp-874': windows874,
|
||||||
|
'win874': windows874,
|
||||||
|
'windows874': windows874,
|
||||||
|
'windows-874': windows874,
|
||||||
|
'cswindows874': windows874,
|
||||||
|
|
||||||
|
'1250': windows1250,
|
||||||
|
'cp1250': windows1250,
|
||||||
|
'cp-1250': windows1250,
|
||||||
|
'win1250': windows1250,
|
||||||
|
'windows1250': windows1250,
|
||||||
|
'windows-1250': windows1250,
|
||||||
|
'cswindows1250': windows1250,
|
||||||
|
|
||||||
|
'1251': windows1251,
|
||||||
|
'cp1251': windows1251,
|
||||||
|
'cp-1251': windows1251,
|
||||||
|
'win1251': windows1251,
|
||||||
|
'windows1251': windows1251,
|
||||||
|
'windows-1251': windows1251,
|
||||||
|
'cswindows1251': windows1251,
|
||||||
|
|
||||||
|
'1252': windows1252,
|
||||||
|
'cp1252': windows1252,
|
||||||
|
'cp-1252': windows1252,
|
||||||
|
'win1252': windows1252,
|
||||||
|
'windows1252': windows1252,
|
||||||
|
'windows-1252': windows1252,
|
||||||
|
'cswindows1252': windows1252,
|
||||||
|
|
||||||
|
'1253': windows1253,
|
||||||
|
'cp1253': windows1253,
|
||||||
|
'cp-1253': windows1253,
|
||||||
|
'win1253': windows1253,
|
||||||
|
'windows1253': windows1253,
|
||||||
|
'windows-1253': windows1253,
|
||||||
|
'cswindows1253': windows1253,
|
||||||
|
|
||||||
|
'1254': windows1254,
|
||||||
|
'cp1254': windows1254,
|
||||||
|
'cp-1254': windows1254,
|
||||||
|
'win1254': windows1254,
|
||||||
|
'windows1254': windows1254,
|
||||||
|
'windows-1254': windows1254,
|
||||||
|
'cswindows1254': windows1254,
|
||||||
|
|
||||||
|
'1255': windows1255,
|
||||||
|
'cp1255': windows1255,
|
||||||
|
'cp-1255': windows1255,
|
||||||
|
'win1255': windows1255,
|
||||||
|
'windows1255': windows1255,
|
||||||
|
'windows-1255': windows1255,
|
||||||
|
'cswindows1255': windows1255,
|
||||||
|
|
||||||
|
'1256': windows1256,
|
||||||
|
'cp1256': windows1256,
|
||||||
|
'cp-1256': windows1256,
|
||||||
|
'win1256': windows1256,
|
||||||
|
'windows1256': windows1256,
|
||||||
|
'windows-1256': windows1256,
|
||||||
|
'cswindows1256': windows1256,
|
||||||
|
|
||||||
|
'1257': windows1257,
|
||||||
|
'cp1257': windows1257,
|
||||||
|
'cp-1257': windows1257,
|
||||||
|
'win1257': windows1257,
|
||||||
|
'windows1257': windows1257,
|
||||||
|
'windows-1257': windows1257,
|
||||||
|
|
||||||
|
'1258': windows1258,
|
||||||
|
'cp1258': windows1258,
|
||||||
|
'cp-1258': windows1258,
|
||||||
|
'win1258': windows1258,
|
||||||
|
'windows1258': windows1258,
|
||||||
|
'windows-1258': windows1258,
|
||||||
|
|
||||||
|
// euc-jp
|
||||||
|
'euc-jp': eucJp,
|
||||||
|
'eucjp': eucJp,
|
||||||
|
'euc_jp': eucJp,
|
||||||
|
'cseucpkdfmtjapanese': eucJp,
|
||||||
|
|
||||||
|
'euc-kr': eucKr,
|
||||||
|
'euckr': eucKr,
|
||||||
|
'euc_kr': eucKr,
|
||||||
|
'cseuckr': eucKr,
|
||||||
|
|
||||||
|
'949': eucKr,
|
||||||
|
'cp949': eucKr,
|
||||||
|
'cp-949': eucKr,
|
||||||
|
'ms949': eucKr,
|
||||||
|
'windows949': eucKr,
|
||||||
|
'csksc56011987': eucKr,
|
||||||
|
'isoir149': eucKr,
|
||||||
|
'korean': eucKr,
|
||||||
|
'ksc56011987': eucKr,
|
||||||
|
'ksc56011989': eucKr,
|
||||||
|
'ksc5601': eucKr,
|
||||||
|
|
||||||
|
'gbk': gbk,
|
||||||
|
'gb2312': gbk,
|
||||||
|
'gb-2312': gbk,
|
||||||
|
'gb_2312': gbk,
|
||||||
|
'cp936': gbk,
|
||||||
|
'cp-936': gbk,
|
||||||
|
'ms936': gbk,
|
||||||
|
'windows-936': gbk,
|
||||||
|
'csgbk': gbk,
|
||||||
|
|
||||||
|
'gb18030': gbk,
|
||||||
|
'csgb18030': gbk,
|
||||||
|
|
||||||
|
'shift-jis': shiftJis,
|
||||||
|
'shiftjis': shiftJis,
|
||||||
|
'shift_jis': shiftJis,
|
||||||
|
'ms_kanji': shiftJis,
|
||||||
|
'csshiftjis': shiftJis,
|
||||||
|
|
||||||
|
'utf-16': utf16,
|
||||||
|
'utf16': utf16,
|
||||||
|
'utf_16': utf16,
|
||||||
|
'csutf16': utf16,
|
||||||
|
'csutf16be': utf16,
|
||||||
|
'csutf16le': utf16,
|
||||||
|
'utf-16be': utf16,
|
||||||
|
'utf-16le': utf16,
|
||||||
|
|
||||||
|
'utf-32': utf32,
|
||||||
|
'utf32': utf32,
|
||||||
|
'utf_32': utf32,
|
||||||
|
'csutf32': utf32,
|
||||||
|
'csutf32be': utf32,
|
||||||
|
'utf-32be': utf32,
|
||||||
|
'csutf32le': utf32,
|
||||||
|
'utf-32le': utf32
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Default detect list
|
||||||
|
static final defaultDetectOrder = <Encoding>[
|
||||||
|
ascii,
|
||||||
|
eucJp,
|
||||||
|
shiftJis,
|
||||||
|
eucKr,
|
||||||
|
gbk,
|
||||||
|
windows874,
|
||||||
|
latin1,
|
||||||
|
latin2,
|
||||||
|
latin3,
|
||||||
|
latin4,
|
||||||
|
latinCyrillic,
|
||||||
|
latinArabic,
|
||||||
|
latinGreek,
|
||||||
|
latinHebrew,
|
||||||
|
latin5,
|
||||||
|
latin6,
|
||||||
|
latinThai,
|
||||||
|
latin7,
|
||||||
|
latin8,
|
||||||
|
latin9,
|
||||||
|
latin10,
|
||||||
|
];
|
||||||
|
|
||||||
|
Charset._();
|
||||||
|
|
||||||
|
/// Detect encoding of bytedata
|
||||||
|
static Encoding? detect(
|
||||||
|
List<int> bytes, {
|
||||||
|
/// falback encoding
|
||||||
|
Encoding? defaultEncoding,
|
||||||
|
|
||||||
|
/// detect list
|
||||||
|
List<Encoding>? orders,
|
||||||
|
|
||||||
|
/// return utf8 when has U+FEFF BOM, either return utf16
|
||||||
|
bool utf8BOM = true,
|
||||||
|
}) {
|
||||||
|
if (hasUtf16BeBom(bytes)) {
|
||||||
|
return utf8BOM ? utf8 : utf16;
|
||||||
|
}
|
||||||
|
if (hasUtf16LeBom(bytes)) {
|
||||||
|
return utf16;
|
||||||
|
}
|
||||||
|
if (hasUtf32beBom(bytes)) {
|
||||||
|
return utf32;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (Encoding encoding in (orders ?? defaultDetectOrder)) {
|
||||||
|
if (canDecode(encoding, bytes)) {
|
||||||
|
return encoding;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return defaultEncoding;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get Encoding by name
|
||||||
|
static Encoding? getByName(String codeName, [Encoding? defaultEncoding]) {
|
||||||
|
return _charCodeMap[codeName.toLowerCase()] ??
|
||||||
|
Encoding.getByName(codeName) ??
|
||||||
|
defaultEncoding;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Register a name and Encoding pair to map
|
||||||
|
static void register(Encoding encoding, String name, [List<String>? alias]) {
|
||||||
|
_charCodeMap[name.toLowerCase()] = encoding;
|
||||||
|
if (alias != null) {
|
||||||
|
for (String aliaName in alias) {
|
||||||
|
_charCodeMap[aliaName.toLowerCase()] = encoding;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether `encoding` can encode the string
|
||||||
|
static bool canEncode(Encoding? encoding, String char) {
|
||||||
|
if (encoding == null) return false;
|
||||||
|
try {
|
||||||
|
List<int> result = encoding.encode(char);
|
||||||
|
if (encoding is CodePage) {
|
||||||
|
if (result.length != char.length) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} else if (!encoding.name.contains('utf')) {
|
||||||
|
if (result.contains(0xFFFD)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} on FormatException catch (_) {
|
||||||
|
return false;
|
||||||
|
} on ArgumentError catch (_) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether `encoding` can decode the bytedata
|
||||||
|
static bool canDecode(Encoding? encoding, List<int> char) {
|
||||||
|
if (encoding == null) return false;
|
||||||
|
try {
|
||||||
|
String result = encoding.decode(char);
|
||||||
|
if (encoding is CodePage) {
|
||||||
|
if (result.contains('\uFFFD')) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} else if (!encoding.name.contains('utf')) {
|
||||||
|
// TODO(shirne): A better way to judge a invalid character
|
||||||
|
if (result.contains('\uFFFD')) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} on FormatException catch (_) {
|
||||||
|
return false;
|
||||||
|
} on ArgumentError catch (_) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
614
lib/charset/code_page.dart
Normal file
614
lib/charset/code_page.dart
Normal file
|
|
@ -0,0 +1,614 @@
|
||||||
|
import 'dart:convert';
|
||||||
|
import 'dart:typed_data';
|
||||||
|
|
||||||
|
/// The ISO-8859-2/Latin-2 (Eastern European) code page.
|
||||||
|
final CodePage latin2 =
|
||||||
|
CodePage._bmp("latin-2", "$_ascii$_noControls$_top8859_2");
|
||||||
|
|
||||||
|
/// The ISO-8859-3/Latin-3 (South European) code page.
|
||||||
|
final CodePage latin3 =
|
||||||
|
CodePage._bmp("latin-3", "$_ascii$_noControls$_top8859_3");
|
||||||
|
|
||||||
|
/// The ISO-8859-4/Latin-4 (North European) code page.
|
||||||
|
final CodePage latin4 =
|
||||||
|
CodePage._bmp("latin-4", "$_ascii$_noControls$_top8859_4");
|
||||||
|
|
||||||
|
/// The ISO-8859-5/Latin-Cyrillic code page.
|
||||||
|
final CodePage latinCyrillic =
|
||||||
|
CodePage._bmp("cyrillic", "$_ascii$_noControls$_top8859_5");
|
||||||
|
|
||||||
|
/// The ISO-8859-6/Latin-Arabic code page.
|
||||||
|
final CodePage latinArabic =
|
||||||
|
CodePage._bmp("arabic", "$_ascii$_noControls$_top8859_6");
|
||||||
|
|
||||||
|
/// The ISO-8859-7/Latin-Greek code page.
|
||||||
|
final CodePage latinGreek =
|
||||||
|
CodePage._bmp("greek", "$_ascii$_noControls$_top8859_7");
|
||||||
|
|
||||||
|
/// The ISO-8859-7/Latin-Hebrew code page.
|
||||||
|
final CodePage latinHebrew =
|
||||||
|
CodePage._bmp("hebrew", "$_ascii$_noControls$_top8859_8");
|
||||||
|
|
||||||
|
/// The ISO-8859-9/Latin-5 (Turkish) code page.
|
||||||
|
final CodePage latin5 =
|
||||||
|
CodePage._bmp("latin-5", "$_ascii$_noControls$_top8859_9");
|
||||||
|
|
||||||
|
/// The ISO-8859-10/Latin-6 (Nordic) code page.
|
||||||
|
final CodePage latin6 =
|
||||||
|
CodePage._bmp("latin-6", "$_ascii$_noControls$_top8859_10");
|
||||||
|
|
||||||
|
/// The ISO-8859-11/Latin-Thai code page.
|
||||||
|
final CodePage latinThai =
|
||||||
|
CodePage._bmp("tis620", "$_ascii$_noControls$_top8859_11");
|
||||||
|
|
||||||
|
/// The ISO-8859-13/Latin-6 (Baltic Rim) code page.
|
||||||
|
final CodePage latin7 =
|
||||||
|
CodePage._bmp("latin-7", "$_ascii$_noControls$_top8859_13");
|
||||||
|
|
||||||
|
/// The ISO-8859-14/Latin-8 (Celtic) code page.
|
||||||
|
final CodePage latin8 =
|
||||||
|
CodePage._bmp("latin-8", "$_ascii$_noControls$_top8859_14");
|
||||||
|
|
||||||
|
/// The ISO-8859-15/Latin-9 (Western European revised) code page.
|
||||||
|
final CodePage latin9 =
|
||||||
|
CodePage._bmp("latin-9", "$_ascii$_noControls$_top8859_15");
|
||||||
|
|
||||||
|
/// The ISO-8859-16/Latin-10 (South Eastern European) code page.
|
||||||
|
final CodePage latin10 =
|
||||||
|
CodePage._bmp("latin-10", "$_ascii$_noControls$_top8859_16");
|
||||||
|
|
||||||
|
final CodePage windows874 = CodePage._bmp("windows874", "$_ascii$_win874");
|
||||||
|
|
||||||
|
final CodePage windows1250 = CodePage._bmp("windows1250", "$_ascii$_win1250");
|
||||||
|
final CodePage windows1251 = CodePage._bmp("windows1251", "$_ascii$_win1251");
|
||||||
|
final CodePage windows1252 = CodePage._bmp("windows1252", "$_ascii$_win1252");
|
||||||
|
final CodePage windows1253 = CodePage._bmp("windows1253", "$_ascii$_win1253");
|
||||||
|
final CodePage windows1254 = CodePage._bmp("windows1254", "$_ascii$_win1254");
|
||||||
|
final CodePage windows1255 = CodePage._bmp("windows1255", "$_ascii$_win1255");
|
||||||
|
final CodePage windows1256 = CodePage._bmp("windows1256", "$_ascii$_win1256");
|
||||||
|
final CodePage windows1257 = CodePage._bmp("windows1257", "$_ascii$_win1257");
|
||||||
|
final CodePage windows1258 = CodePage._bmp("windows1258", "$_ascii$_win1258");
|
||||||
|
|
||||||
|
final CodePage cp437 = CodePage._bmp("cp437", "$_ascii$_cp437");
|
||||||
|
final CodePage cp737 = CodePage._bmp("cp737", "$_ascii$_cp737");
|
||||||
|
final CodePage cp775 = CodePage._bmp("cp775", "$_ascii$_cp775");
|
||||||
|
final CodePage cp850 = CodePage._bmp("cp850", "$_ascii$_cp850");
|
||||||
|
final CodePage cp852 = CodePage._bmp("cp852", "$_ascii$_cp852");
|
||||||
|
final CodePage cp855 = CodePage._bmp("cp855", "$_ascii$_cp855");
|
||||||
|
final CodePage cp856 = CodePage._bmp("cp856", "$_ascii$_cp856");
|
||||||
|
final CodePage cp857 = CodePage._bmp("cp857", "$_ascii$_cp857");
|
||||||
|
final CodePage cp858 = CodePage._bmp("cp858", "$_ascii$_cp858");
|
||||||
|
final CodePage cp860 = CodePage._bmp("cp860", "$_ascii$_cp860");
|
||||||
|
final CodePage cp861 = CodePage._bmp("cp861", "$_ascii$_cp861");
|
||||||
|
final CodePage cp862 = CodePage._bmp("cp862", "$_ascii$_cp862");
|
||||||
|
final CodePage cp863 = CodePage._bmp("cp863", "$_ascii$_cp863");
|
||||||
|
final CodePage cp864 = CodePage._bmp("cp864", "$_ascii$_cp864");
|
||||||
|
final CodePage cp865 = CodePage._bmp("cp865", "$_ascii$_cp865");
|
||||||
|
final CodePage cp866 = CodePage._bmp("cp866", "$_ascii$_cp866");
|
||||||
|
final CodePage cp869 = CodePage._bmp("cp869", "$_ascii$_cp869");
|
||||||
|
final CodePage cp922 = CodePage._bmp("cp922", "$_ascii$_noControls$_cp922");
|
||||||
|
final CodePage cp1046 = CodePage._bmp("cp1046", "$_ascii$_cp1046");
|
||||||
|
final CodePage cp1124 = CodePage._bmp("cp1124", "$_ascii$_noControls$_cp1124");
|
||||||
|
final CodePage cp1125 = CodePage._bmp("cp1125", "$_ascii$_cp1125");
|
||||||
|
final CodePage cp1129 = CodePage._bmp("cp1129", "$_ascii$_noControls$_cp1129");
|
||||||
|
final CodePage cp1133 = CodePage._bmp("cp1133", "$_ascii$_noControls$_cp1133");
|
||||||
|
final CodePage cp1161 = CodePage._bmp("cp1161", "$_ascii$_cp1161");
|
||||||
|
final CodePage cp1162 = CodePage._bmp("cp1162", "$_ascii$_cp1162");
|
||||||
|
final CodePage cp1163 = CodePage._bmp("cp1163", "$_ascii$_noControls$_cp1163");
|
||||||
|
|
||||||
|
/// Characters in ISO-8859-2 above the ASCII and top control characters.
|
||||||
|
const _top8859_2 = "\xa0Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\xadŽŻ°ą˛ł´ľśˇ¸šşťź˝žż"
|
||||||
|
"ŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢß"
|
||||||
|
"ŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
|
||||||
|
|
||||||
|
/// Characters in ISO-8859-3 above the ASCII and top control characters.
|
||||||
|
const _top8859_3 = "\xa0Ħ˘£\uFFFD¤Ĥ§¨İŞĞĴ\xad\uFFFDŻ°ħ²³´µĥ·¸ışğĵ½\uFFFDż"
|
||||||
|
"ÀÁÂ\uFFFDÄĊĈÇÈÉÊËÌÍÎÏ\uFFFDÑÒÓÔĠÖ×ĜÙÚÛÜŬŜß"
|
||||||
|
"àáâ\uFFFDäċĉçèéêëìíîï\uFFFDñòóôġö÷ĝùúûüŭŝ˙";
|
||||||
|
|
||||||
|
/// Characters in ISO-8859-4 above the ASCII and top control characters.
|
||||||
|
const _top8859_4 = "\xa0ĄĸŖ¤ĨĻ§¨ŠĒĢŦ\xadŽ¯°ą˛ŗ´ĩļˇ¸šēģŧŊžŋ"
|
||||||
|
"ĀÁÂÃÄÅÆĮČÉĘËĖÍÎĪĐŅŌĶÔÕÖ×ØŲÚÛÜŨŪß"
|
||||||
|
"āáâãäåæįčéęëėíîīđņōķôõö÷øųúûüũū˙";
|
||||||
|
|
||||||
|
/// Characters in ISO-8859-5 above the ASCII and top control characters.
|
||||||
|
const _top8859_5 = "\xa0ЁЂЃЄЅІЇЈЉЊЋЌ\xadЎЏАБВГДЕЖЗИЙКЛМНОП"
|
||||||
|
"РСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмноп"
|
||||||
|
"рстуфхцчшщъыьэюя№ёђѓєѕіїјљњћќ§ўџ";
|
||||||
|
|
||||||
|
/// Characters in ISO-8859-6 above the ASCII and top control characters.
|
||||||
|
const _top8859_6 = "\xa0\uFFFD\uFFFD\uFFFD¤\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\u060c\xad\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\u061b\uFFFD\uFFFD\uFFFD\u061f"
|
||||||
|
"\uFFFD\u0621\u0622\u0623\u0624\u0625\u0626\u0627"
|
||||||
|
"\u0628\u0629\u062a\u062b\u062c\u062d\u062e\u062f"
|
||||||
|
"\u0630\u0631\u0632\u0633\u0634\u0635\u0636\u0637"
|
||||||
|
"\u0638\u0639\u063a\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\u0640\u0641\u0642\u0643\u0644\u0645\u0646\u0647"
|
||||||
|
"\u0648\u0649\u064a\u064b\u064c\u064d\u064e\u064f"
|
||||||
|
"\u0650\u0651\u0652\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD";
|
||||||
|
|
||||||
|
/// Characters in ISO-8859-7 above the ASCII and top control characters.
|
||||||
|
const _top8859_7 = "\xa0‘’£€₯¦§¨©ͺ«¬\xad\uFFFD―°±²³΄΅Ά·ΈΉΊ»Ό½ΎΏ"
|
||||||
|
"ΐΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡ\uFFFDΣΤΥΦΧΨΩΪΫάέήί"
|
||||||
|
"ΰαβγδεζηθικλμνξοπρςστυφχψωϊϋόύώ\uFFFD";
|
||||||
|
|
||||||
|
/// Characters in ISO-8859-8 above the ASCII and top control characters.
|
||||||
|
const _top8859_8 = "\xa0\uFFFD¢£¤¥¦§¨©×«¬\xad®¯°±²³´µ¶·¸¹÷»¼½¾\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD‗"
|
||||||
|
"\u05d0\u05d1\u05d2\u05d3\u05d4\u05d5\u05d6\u05d7"
|
||||||
|
"\u05d8\u05d9\u05da\u05db\u05dc\u05dd\u05de\u05df"
|
||||||
|
"\u05e0\u05e1\u05e2\u05e3\u05e4\u05e5\u05e6\u05e7"
|
||||||
|
"\u05e8\u05e9\u05ea\uFFFD\uFFFD\u200e\u200f\uFFFD";
|
||||||
|
|
||||||
|
/// Characters in ISO-8859-9 above the ASCII and top control characters.
|
||||||
|
const _top8859_9 = "\xa0¡¢£¤¥¦§¨©ª«¬\xad®¯°±²³´µ¶·¸¹º»¼½¾¿"
|
||||||
|
"ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏĞÑÒÓÔÕÖ×ØÙÚÛÜİŞß"
|
||||||
|
"àáâãäåæçèéêëìíîïğñòóôõö÷øùúûüışÿ";
|
||||||
|
|
||||||
|
/// Characters in ISO-8859-10 above the ASCII and top control characters.
|
||||||
|
const _top8859_10 = "\xa0ĄĒĢĪĨĶ§ĻĐŠŦŽ\xadŪŊ°ąēģīĩķ·ļđšŧž―ūŋ"
|
||||||
|
"ĀÁÂÃÄÅÆĮČÉĘËĖÍÎÏÐŅŌÓÔÕÖŨØŲÚÛÜÝÞß"
|
||||||
|
"āáâãäåæįčéęëėíîïðņōóôõöũøųúûüýþĸ";
|
||||||
|
|
||||||
|
/// Characters in ISO-8859-11 above the ASCII and top control characters.
|
||||||
|
const _top8859_11 = "\xa0กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟ"
|
||||||
|
"ภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู\uFFFD\uFFFD\uFFFD\uFFFD฿"
|
||||||
|
"เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛\uFFFD\uFFFD\uFFFD\uFFFD";
|
||||||
|
|
||||||
|
/// Characters in ISO-8859-13 above the ASCII and top control characters.
|
||||||
|
const _top8859_13 = "\xa0”¢£¤„¦§Ø©Ŗ«¬\xad®Æ°±²³“µ¶·ø¹ŗ»¼½¾æ"
|
||||||
|
"ĄĮĀĆÄÅĘĒČÉŹĖĢĶĪĻŠŃŅÓŌÕÖ×ŲŁŚŪÜŻŽß"
|
||||||
|
"ąįāćäåęēčéźėģķīļšńņóōõö÷ųłśūüżž’";
|
||||||
|
|
||||||
|
/// Characters in ISO-8859-14 above the ASCII and top control characters.
|
||||||
|
const _top8859_14 = "\xa0Ḃḃ£ĊċḊ§Ẁ©ẂḋỲ\xad®ŸḞḟĠġṀṁ¶ṖẁṗẃṠỳẄẅṡ"
|
||||||
|
"ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏŴÑÒÓÔÕÖṪØÙÚÛÜÝŶß"
|
||||||
|
"àáâãäåæçèéêëìíîïŵñòóôõöṫøùúûüýŷÿ";
|
||||||
|
|
||||||
|
/// Characters in ISO-8859-15 above the ASCII and top control characters.
|
||||||
|
const _top8859_15 = "\xa0¡¢£€¥Š§š©ª«¬\xad®¯°±²³Žµ¶·ž¹º»ŒœŸ¿"
|
||||||
|
"ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞß"
|
||||||
|
"àáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ";
|
||||||
|
|
||||||
|
/// Characters in ISO-8859-16 above the ASCII and top control characters.
|
||||||
|
const _top8859_16 = "\xa0ĄąŁ€„Š§š©Ș«Ź\xadźŻ°±ČłŽ”¶·žčș»ŒœŸż"
|
||||||
|
"ÀÁÂĂÄĆÆÇÈÉÊËÌÍÎÏĐŃÒÓÔŐÖŚŰÙÚÛÜĘȚß"
|
||||||
|
"àáâăäćæçèéêëìíîïđńòóôőöśűùúûüęțÿ";
|
||||||
|
|
||||||
|
const _win874 =
|
||||||
|
"€\uFFFD\uFFFD\uFFFD\uFFFD…\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD‘’“”•–—\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
" กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛\uFFFD\uFFFD\uFFFD\uFFFD";
|
||||||
|
|
||||||
|
const _win1250 =
|
||||||
|
"€\uFFFD‚\uFFFD„…†‡\uFFFD‰Š‹ŚŤŽŹ\uFFFD‘’“”•–—\uFFFD™š›śťžź ˇ˘Ł¤Ą¦§¨©Ş«¬®Ż°"
|
||||||
|
"±˛ł´µ¶·¸ąş»Ľ˝ľżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷ř"
|
||||||
|
"ůúűüýţ˙";
|
||||||
|
|
||||||
|
const _win1251 =
|
||||||
|
"ЂЃ‚ѓ„…†‡€‰Љ‹ЊЌЋЏђ‘’“”•–—\uFFFD™љ›њќћџ ЎўЈ¤Ґ¦§Ё©Є«¬®Ї°±Ііґµ¶·ё№є»јЅѕ"
|
||||||
|
"їАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя";
|
||||||
|
|
||||||
|
const _win1252 =
|
||||||
|
"€\uFFFD‚ƒ„…†‡ˆ‰Š‹Œ\uFFFDŽ\uFFFD\uFFFD‘’“”•–—˜™š›œ\uFFFDžŸ ¡¢£¤¥¦§¨©ª«¬®¯°"
|
||||||
|
"±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷"
|
||||||
|
"øùúûüýþÿ";
|
||||||
|
const _win1253 =
|
||||||
|
"€\uFFFD‚ƒ„…†‡\uFFFD‰\uFFFD‹\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD‘’“”•–—\uFFFD™"
|
||||||
|
"\uFFFD›\uFFFD\uFFFD\uFFFD\uFFFD ΅Ά£¤¥¦§¨©\uFFFD«¬®―°±²³΄µ¶·ΈΉΊ»Ό½ΎΏΐΑΒΓΔ"
|
||||||
|
"ΕΖΗΘΙΚΛΜΝΞΟΠΡ\uFFFDΣΤΥΦΧΨΩΪΫάέήίΰαβγδεζηθικλμνξοπρςστυφχψωϊϋόύώ\uFFFD";
|
||||||
|
const _win1254 =
|
||||||
|
"€\uFFFD‚ƒ„…†‡ˆ‰Š‹Œ\uFFFD\uFFFD\uFFFD\uFFFD‘’“”•–—˜™š›œ\uFFFD\uFFFDŸ ¡¢£¤¥"
|
||||||
|
"¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏĞÑÒÓÔÕÖ×ØÙÚÛÜİŞßàáâãäåæçèéêëìíîï"
|
||||||
|
"ğñòóôõö÷øùúûüışÿ";
|
||||||
|
const _win1255 =
|
||||||
|
"€\uFFFD‚ƒ„…†‡ˆ‰\uFFFD‹\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD‘’“”•–—˜™\uFFFD›"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD ¡¢£₪¥¦§¨©×«¬®¯°±²³´µ¶·¸¹÷»¼½¾¿ְֱֲֳִֵֶַָֹֺֻּֽ־ֿ׀ׁׂ׃װױײ׳״"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFDאבגדהוזחטיךכלםמןנסעףפץצקרשת"
|
||||||
|
"\uFFFD\uFFFD\uFFFD";
|
||||||
|
const _win1256 = "€پ‚ƒ„…†‡ˆ‰ٹ‹Œچژڈگ‘’“”•–—ک™ڑ›œں ،¢£¤¥¦§¨©ھ«¬®¯°±²³´µ¶"
|
||||||
|
"·¸¹؛»¼½¾؟ہءآأؤإئابةتثجحخدذرزسشصض×طظعغـفقكàلâمنهوçèéêëىيîïًٌٍَôُِ÷ّùْûü"
|
||||||
|
"ے";
|
||||||
|
const _win1257 = "€\uFFFD‚\uFFFD„…†‡\uFFFD‰\uFFFD‹\uFFFD¨ˇ¸\uFFFD‘’“”•–—\uFFFD™"
|
||||||
|
"\uFFFD›\uFFFD¯˛\uFFFD \uFFFD¢£¤\uFFFD¦§Ø©Ŗ«¬®Æ°±²³´µ¶·ø¹ŗ»¼½¾æ"
|
||||||
|
"ĄĮĀĆÄÅĘĒČÉŹĖĢĶĪĻŠŃŅÓŌÕÖ×ŲŁŚŪÜŻŽßąįāćäåęēčéźėģķīļšńņóōõö÷ųłśūüżž˙";
|
||||||
|
const _win1258 =
|
||||||
|
"€\uFFFD‚ƒ„…†‡ˆ‰\uFFFD‹Œ\uFFFD\uFFFD\uFFFD\uFFFD‘’“”•–—˜™\uFFFD›œ"
|
||||||
|
"\uFFFD\uFFFDŸ ¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂĂÄÅÆÇÈÉÊË̀ÍÎÏĐÑ̉ÓÔƠÖ"
|
||||||
|
"×ØÙÚÛÜỮßàáâăäåæçèéêë́íîïđṇ̃óôơö÷øùúûüư₫ÿ";
|
||||||
|
|
||||||
|
const _cp437 =
|
||||||
|
"ÇüéâäàåçêëèïîìÄÅÉæÆôöòûùÿÖÜ¢£¥₧ƒáíóúñѪº¿⌐¬½¼¡«»░▒▓│┤╡╢╖╕╣║╗╝╜╛┐"
|
||||||
|
"└┴┬├─┼╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀αßΓπΣσµτΦΘΩδ∞φε∩≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ ";
|
||||||
|
const _cp737 =
|
||||||
|
"ΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩαβγδεζηθικλμνξοπρσςτυφχψ░▒▓│┤╡╢╖╕╣║╗╝╜╛┐"
|
||||||
|
"└┴┬├─┼╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀ωάέήϊίόύϋώΆΈΉΊΌΎΏ±≥≤ΪΫ÷≈°∙·√ⁿ²■ ";
|
||||||
|
const _cp775 =
|
||||||
|
"ĆüéāäģåćłēŖŗīŹÄÅÉæÆōöĢ¢ŚśÖÜø£ØפĀĪóŻżź”¦©®¬½¼Ł«»░▒▓│┤ĄČĘĖ╣║╗╝ĮŠ┐"
|
||||||
|
"└┴┬├─┼ŲŪ╚╔╩╦╠═╬Žąčęėįšųūž┘┌█▄▌▐▀ÓßŌŃõÕµńĶķĻļņĒŅ’±“¾¶§÷„°∙·¹³²■ ";
|
||||||
|
const _cp850 =
|
||||||
|
"ÇüéâäàåçêëèïîìÄÅÉæÆôöòûùÿÖÜø£Ø׃áíóúñѪº¿®¬½¼¡«»░▒▓│┤ÁÂÀ©╣║╗╝¢¥┐"
|
||||||
|
"└┴┬├─┼ãÃ╚╔╩╦╠═╬¤ðÐÊËÈıÍÎÏ┘┌█▄¦Ì▀ÓßÔÒõÕµþÞÚÛÙýݯ´±‗¾¶§÷¸°¨·¹³²■ ";
|
||||||
|
const _cp852 =
|
||||||
|
"ÇüéâäůćçłëŐőîŹÄĆÉĹĺôöĽľŚśÖÜŤťŁ×čáíóúĄąŽžĘ꬟Ⱥ«»░▒▓│┤ÁÂĚŞ╣║╗╝Żż┐"
|
||||||
|
"└┴┬├─┼Ăă╚╔╩╦╠═╬¤đĐĎËďŇÍÎě┘┌█▄ŢŮ▀ÓßÔŃńňŠšŔÚŕŰýÝţ´˝˛ˇ˘§÷¸°¨˙űŘř■ ";
|
||||||
|
const _cp855 =
|
||||||
|
"ђЂѓЃёЁєЄѕЅіІїЇјЈљЉњЊћЋќЌўЎџЏюЮъЪаАбБцЦдДеЕфФгГ«»░▒▓│┤хХиИ╣║╗╝йЙ┐"
|
||||||
|
"└┴┬├─┼кК╚╔╩╦╠═╬¤лЛмМнНоОп┘┌█▄Пя▀ЯрРсСтТуУжЖвВьЬ№ыЫзЗшШэЭщЩчЧ§■ ";
|
||||||
|
const _cp856 =
|
||||||
|
"אבגדהוזחטיךכלםמןנסעףפץצקרשת\uFFFD£\uFFFD×\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD®¬½¼\uFFFD«»░▒▓│┤\uFFFD\uFFFD"
|
||||||
|
"\uFFFD©╣║╗╝¢¥┐└┴┬├─┼\uFFFD\uFFFD╚╔╩╦╠═╬¤\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD┘┌█▄¦\uFFFD▀\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFDµ\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD¯´±‗¾¶§÷¸°¨·¹³²■ ";
|
||||||
|
const _cp857 =
|
||||||
|
"ÇüéâäàåçêëèïîıÄÅÉæÆôöòûùİÖÜø£ØŞşáíóúñÑĞ𿮬½¼¡«»░▒▓│┤ÁÂÀ©╣║╗╝¢¥┐└┴┬├─┼"
|
||||||
|
"ãÃ╚╔╩╦╠═╬¤ºªÊËÈ\uFFFDÍÎÏ┘┌█▄¦Ì▀ÓßÔÒõÕµ\uFFFD×ÚÛÙìÿ¯´±\uFFFD¾¶§÷¸°¨·¹³²■ ";
|
||||||
|
const _cp858 =
|
||||||
|
"ÇüéâäàåçêëèïîìÄÅÉæÆôöòûùÿÖÜø£Ø׃áíóúñѪº¿®¬½¼¡«»░▒▓│┤ÁÂÀ©╣║╗╝¢¥┐└┴┬├─┼"
|
||||||
|
"ãÃ╚╔╩╦╠═╬¤ðÐÊËÈ€ÍÎÏ┘┌█▄¦Ì▀ÓßÔÒõÕµþÞÚÛÙýݯ´±‗¾¶§÷¸°¨·¹³²■ ";
|
||||||
|
const _cp860 =
|
||||||
|
"ÇüéâãàÁçêÊèÍÔìÃÂÉÀÈôõòÚùÌÕÜ¢£Ù₧ÓáíóúñѪº¿Ò¬½¼¡«»░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼"
|
||||||
|
"╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀αßΓπΣσµτΦΘΩδ∞φε∩≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ ";
|
||||||
|
const _cp861 =
|
||||||
|
"ÇüéâäàåçêëèÐðÞÄÅÉæÆôöþûÝýÖÜø£Ø₧ƒáíóúÁÍÓÚ¿⌐¬½¼¡«»░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼"
|
||||||
|
"╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀αßΓπΣσµτΦΘΩδ∞φε∩≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ ";
|
||||||
|
const _cp862 =
|
||||||
|
"אבגדהוזחטיךכלםמןנסעףפץצקרשת¢£¥₧ƒáíóúñѪº¿⌐¬½¼¡«»░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼"
|
||||||
|
"╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀αßΓπΣσµτΦΘΩδ∞φε∩≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ ";
|
||||||
|
const _cp863 =
|
||||||
|
"ÇüéâÂà¶çêëèïî‗À§ÉÈÊôËÏûù¤ÔÜ¢£ÙÛƒ¦´óú¨¸³¯Î⌐¬½¼¾«»░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼"
|
||||||
|
"╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀αßΓπΣσµτΦΘΩδ∞φε∩≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ ";
|
||||||
|
const _cp864 = "°·∙√▒─│┼┤┬├┴┐┌└┘β∞φ±½¼≈«»ﻷﻸ\uFFFD\uFFFDﻻﻼ\uFFFD ﺂ£¤ﺄ\uFFFD"
|
||||||
|
"\uFFFDﺎﺏﺕﺙ،ﺝﺡﺥ٠١٢٣٤٥٦٧٨٩ﻑ؛ﺱﺵﺹ؟¢ﺀ"
|
||||||
|
"ﺁﺃﺅﻊﺋﺍﺑﺓﺗﺛﺟﺣﺧﺩﺫﺭﺯﺳﺷﺻﺿﻁﻅﻋﻏ¦¬÷×ﻉـﻓﻗﻛﻟﻣﻧﻫﻭﻯﻳﺽﻌﻎﻍﻡﹽّﻥﻩﻬﻰﻲﻐﻕﻵﻶﻝﻙﻱ■\uFFFD";
|
||||||
|
const _cp865 =
|
||||||
|
"ÇüéâäàåçêëèïîìÄÅÉæÆôöòûùÿÖÜø£Ø₧ƒáíóúñѪº¿⌐¬½¼¡«¤░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼"
|
||||||
|
"╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀αßΓπΣσµτΦΘΩδ∞φε∩≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ ";
|
||||||
|
const _cp866 =
|
||||||
|
"АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмноп░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼"
|
||||||
|
"╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀рстуфхцчшщъыьэюяЁёЄєЇїЎў°∙·√№¤■ ";
|
||||||
|
const _cp869 =
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFDΆ\uFFFD·¬¦‘’Έ―ΉΊΪΌ\uFFFD\uFFFD"
|
||||||
|
"ΎΫ©Ώ²³ά£έήίϊΐόύΑΒΓΔΕΖΗ½ΘΙ«»░▒▓│┤ΚΛΜΝ╣║╗╝ΞΟ┐└┴┬├─┼ΠΡ╚╔╩╦╠═╬ΣΤΥΦΧΨΩαβγ┘"
|
||||||
|
"┌█▄δε▀ζηθικλμνξοπρσςτ΄±υφχ§ψ΅°¨ωϋΰώ■ ";
|
||||||
|
const _cp922 =
|
||||||
|
" ¡¢£¤¥¦§¨©ª«¬®‾°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏŠÑÒÓÔÕÖ×ØÙÚÛÜÝŽß"
|
||||||
|
"àáâãäåæçèéêëìíîïšñòóôõö÷øùúûüýžÿ";
|
||||||
|
const _cp1046 =
|
||||||
|
"ﺈ×÷ﹱ■│─┐┌└┘ﹹﹻﹽﹿﹷﺊﻰﻳﻲﻎﻏﻐﻶﻸﻺﻼ ¤ﺋﺑﺗﺛﺟﺣ،ﺧﺳ٠١٢٣٤٥٦٧٨٩ﺷ؛ﺻﺿﻊ؟ﻋءآأؤإئابة"
|
||||||
|
"تثجحخدذرزسشصضطﻇعغﻌﺂﺄﺎﻓـفقكلمنهوىيًٌٍَُِّْﻗﻛﻟﻵﻷﻹﻻﻣﻧﻬﻩ\uFFFD";
|
||||||
|
const _cp1124 =
|
||||||
|
" ЁЂҐЄЅІЇЈЉЊЋЌЎЏАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмно"
|
||||||
|
"прстуфхцчшщъыьэюя№ёђґєѕіїјљњћќ§ўџ";
|
||||||
|
const _cp1125 =
|
||||||
|
"АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмноп░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼"
|
||||||
|
"╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀рстуфхцчшщъыьэюяЁёҐґЄєІіЇї·√№¤■ ";
|
||||||
|
const _cp1129 =
|
||||||
|
" ¡¢£¤¥¦§œ©ª«¬®¯°±²³Ÿµ¶·Œ¹º»¼½¾¿ÀÁÂĂÄÅÆÇÈÉÊË̀ÍÎÏĐÑ̉ÓÔƠÖ×ØÙÚÛÜỮß"
|
||||||
|
"àáâăäåæçèéêë́íîïđṇ̃óôơö÷øùúûüư₫ÿ";
|
||||||
|
const _cp1133 =
|
||||||
|
" ກຂຄງຈສຊຍດຕຖທນບປຜຝພຟມຢຣລວຫອຮ\uFFFD\uFFFD\uFFFDຯະາຳິີຶືຸູຼັົຽ\uFFFD\uFFFD\uFFFD"
|
||||||
|
"ເແໂໃໄ່້໊໋໌ໍໆ\uFFFDໜໝ₭\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD໐໑໒໓໔໕໖໗໘໙\uFFFD\uFFFD¢¬¦\uFFFD";
|
||||||
|
const _cp1161 =
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD่กขฃคฅฆง"
|
||||||
|
"จฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู้๊๋€฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛¢¬¦ ";
|
||||||
|
const _cp1162 =
|
||||||
|
"€…‘’“”•–— กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภ"
|
||||||
|
"มยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู\uFFFD\uFFFD\uFFFD\uFFFD฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD";
|
||||||
|
const _cp1163 =
|
||||||
|
" ¡¢£€¥¦§œ©ª«¬®¯°±²³Ÿµ¶·Œ¹º»¼½¾¿ÀÁÂĂÄÅÆÇÈÉÊË̀ÍÎÏĐÑ̉ÓÔƠÖ×ØÙÚÛÜỮ"
|
||||||
|
"ßàáâăäåæçèéêë́íîïđṇ̃óôơö÷øùúûüư₫ÿ";
|
||||||
|
|
||||||
|
const _noControls = "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"
|
||||||
|
"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD";
|
||||||
|
|
||||||
|
const _controls = "\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007"
|
||||||
|
"\b\t\n\u000b\f\r\u000e\u000f\u0010\u0011"
|
||||||
|
"\u0012\u0013\u0014\u0015\u0016\u0017\u0018"
|
||||||
|
"\u0019\u001a\u001b\u001c\u001d\u001e\u001f";
|
||||||
|
|
||||||
|
/// ASCII characters with control characters. Shared by many code pages.
|
||||||
|
const _ascii = "$_controls"
|
||||||
|
r""" !"#$%&'()*+,-./0123456789:;<=>?"""
|
||||||
|
r"@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_"
|
||||||
|
"`abcdefghijklmnopqrstuvwxyz{|}~\uFFFD";
|
||||||
|
|
||||||
|
/// A mapping between bytes and characters.
|
||||||
|
///
|
||||||
|
/// A code page is a way to map bytes to character.
|
||||||
|
/// As such, it can only represent 256 different characters.
|
||||||
|
class CodePage extends Encoding {
|
||||||
|
@override
|
||||||
|
final CodePageDecoder decoder;
|
||||||
|
@override
|
||||||
|
final String name;
|
||||||
|
CodePageEncoder? _encoder;
|
||||||
|
|
||||||
|
/// Creates a code page with the given name and characters.
|
||||||
|
///
|
||||||
|
/// The [characters] string must contain 256 code points (runes)
|
||||||
|
/// in the order of the bytes representing them.
|
||||||
|
///
|
||||||
|
/// Any byte not defined by the code page should have a
|
||||||
|
/// U+FFFD (invalid character) code point at its place in
|
||||||
|
/// [characters].
|
||||||
|
///
|
||||||
|
/// The name is used by [Encoding.name].
|
||||||
|
factory CodePage(String name, String characters) = CodePage._general;
|
||||||
|
|
||||||
|
/// Creates a code page with the characters of [characters].
|
||||||
|
///
|
||||||
|
/// The [characters] must contain precisely 256 characters (code points).
|
||||||
|
///
|
||||||
|
/// A U+FFFD (invalid character) entry in [character] means that the
|
||||||
|
/// corresponding byte does not have a definition in this code page.
|
||||||
|
CodePage._general(this.name, String characters)
|
||||||
|
: decoder = _createDecoder(characters);
|
||||||
|
|
||||||
|
/// Creates a code page with characters from the basic multilingual plane.
|
||||||
|
///
|
||||||
|
/// The basic multilingual plane (BMP) contains the first 65536 code points.
|
||||||
|
/// As such, each character can be represented by a single UTF-16 code unit,
|
||||||
|
/// which makes some operations more efficient.
|
||||||
|
///
|
||||||
|
/// The [characters] must contain precisely 256 code points from the BMP
|
||||||
|
/// which means that it should have length 256 and not contain any surrogates.
|
||||||
|
///
|
||||||
|
/// A U+FFFD (invalid character) entry in [character] means that the
|
||||||
|
/// corresponding byte does not have a definition in this code page.
|
||||||
|
CodePage._bmp(this.name, String characters)
|
||||||
|
: decoder = _BmpCodePageDecoder(characters);
|
||||||
|
|
||||||
|
/// The character associated with a particular byte in this code page.
|
||||||
|
///
|
||||||
|
/// The [byte] must be in the range 0..255.
|
||||||
|
/// The returned value should be a Unicode scalar value
|
||||||
|
/// (a non-surrogate code point).
|
||||||
|
///
|
||||||
|
/// If a code page does not have a defined character for a particular
|
||||||
|
/// byte, it should return the Unicode invalid character (U+FFFD)
|
||||||
|
/// instad.
|
||||||
|
int operator [](int byte) => decoder._char(byte);
|
||||||
|
|
||||||
|
/// Encodes [input] using `encoder.convert`.
|
||||||
|
@override
|
||||||
|
Uint8List encode(String input, {int? invalidCharacter}) =>
|
||||||
|
encoder.convert(input, invalidCharacter: invalidCharacter);
|
||||||
|
|
||||||
|
/// Decodes [bytes] using `encoder.convert`.
|
||||||
|
@override
|
||||||
|
String decode(List<int> bytes, {bool allowInvalid = false}) =>
|
||||||
|
decoder.convert(bytes, allowInvalid: allowInvalid);
|
||||||
|
|
||||||
|
@override
|
||||||
|
CodePageEncoder get encoder => _encoder ??= decoder._createEncoder();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A code page decoder, converts from bytes to characters.
|
||||||
|
///
|
||||||
|
/// A code page assigns characters to a subset of byte values.
|
||||||
|
/// The decoder converts those bytes back to their characters.
|
||||||
|
abstract class CodePageDecoder implements Converter<List<int>, String> {
|
||||||
|
/// Decodes a sequence of bytes into a string using a code page.
|
||||||
|
///
|
||||||
|
/// The code page assigns one character to each byte.
|
||||||
|
/// Values in [input] must be bytes (integers in the range 0..255).
|
||||||
|
///
|
||||||
|
/// If [allowInvalid] is true, non-byte values in [input],
|
||||||
|
/// or byte values not defined as a character in the code page,
|
||||||
|
/// are emitted as U+FFFD (the Unicode invalid character).
|
||||||
|
/// If not true, the bytes must be calid and defined characters.
|
||||||
|
@override
|
||||||
|
String convert(List<int> input, {bool allowInvalid = false});
|
||||||
|
|
||||||
|
CodePageEncoder _createEncoder();
|
||||||
|
int _char(int byte);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Creates a decoder from [characters].
|
||||||
|
///
|
||||||
|
/// Recognizes if [characters] contains only characters in the BMP,
|
||||||
|
/// and creates a [BmpCodePageDecoder] in that case.
|
||||||
|
CodePageDecoder _createDecoder(String characters) {
|
||||||
|
var result = Uint32List(256);
|
||||||
|
var i = 0;
|
||||||
|
var allChars = 0;
|
||||||
|
for (var char in characters.runes) {
|
||||||
|
if (i >= 256) {
|
||||||
|
throw ArgumentError.value(
|
||||||
|
characters, "characters", "Must contain 256 characters");
|
||||||
|
}
|
||||||
|
result[i] = char;
|
||||||
|
allChars |= char;
|
||||||
|
}
|
||||||
|
if (i < 256) {
|
||||||
|
throw ArgumentError.value(
|
||||||
|
characters, "characters", "Must contain 256 characters");
|
||||||
|
}
|
||||||
|
if (allChars <= 0xFFFF) {
|
||||||
|
// It's in the BMP.
|
||||||
|
return _BmpCodePageDecoder(characters);
|
||||||
|
}
|
||||||
|
return _NonBmpCodePageDecoder._(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Code page with non-BMP characters.
|
||||||
|
class _NonBmpCodePageDecoder extends Converter<List<int>, String>
|
||||||
|
implements CodePageDecoder {
|
||||||
|
final Uint32List _characters;
|
||||||
|
_NonBmpCodePageDecoder(String characters) : this._(_buildMapping(characters));
|
||||||
|
_NonBmpCodePageDecoder._(this._characters);
|
||||||
|
|
||||||
|
@override
|
||||||
|
int _char(int byte) => _characters[byte];
|
||||||
|
|
||||||
|
static Uint32List _buildMapping(String characters) {
|
||||||
|
var result = Uint32List(256);
|
||||||
|
var i = 0;
|
||||||
|
for (var char in characters.runes) {
|
||||||
|
if (i >= 256) {
|
||||||
|
throw ArgumentError.value(
|
||||||
|
characters, "characters", "Must contain 256 characters");
|
||||||
|
}
|
||||||
|
result[i++] = char;
|
||||||
|
}
|
||||||
|
if (i < 256) {
|
||||||
|
throw ArgumentError.value(
|
||||||
|
characters, "characters", "Must contain 256 characters");
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
CodePageEncoder _createEncoder() {
|
||||||
|
var result = <int, int>{};
|
||||||
|
for (var i = 0; i < 256; i++) {
|
||||||
|
var char = _characters[i];
|
||||||
|
if (char != 0xFFFD) {
|
||||||
|
result[char] = i;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return CodePageEncoder._(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
String convert(List<int> input, {bool allowInvalid = false}) {
|
||||||
|
var buffer = Uint32List(input.length);
|
||||||
|
for (var i = 0; i < input.length; i++) {
|
||||||
|
var byte = input[i];
|
||||||
|
if (byte & 0xff != byte) throw FormatException("Not a byte", input, i);
|
||||||
|
buffer[i] = _characters[byte];
|
||||||
|
}
|
||||||
|
return String.fromCharCodes(buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class _BmpCodePageDecoder extends Converter<List<int>, String>
|
||||||
|
implements CodePageDecoder {
|
||||||
|
final String _characters;
|
||||||
|
_BmpCodePageDecoder(String characters) : _characters = characters {
|
||||||
|
if (characters.length != 256) {
|
||||||
|
throw ArgumentError.value(characters, "characters",
|
||||||
|
"Must contain 256 characters. Was ${characters.length}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
int _char(int byte) => _characters.codeUnitAt(byte);
|
||||||
|
|
||||||
|
@override
|
||||||
|
String convert(List<int> bytes, {bool allowInvalid = false}) {
|
||||||
|
if (allowInvalid) return _convertAllowInvalid(bytes);
|
||||||
|
var count = bytes.length;
|
||||||
|
var codeUnits = Uint16List(count);
|
||||||
|
for (var i = 0; i < count; i++) {
|
||||||
|
var byte = bytes[i];
|
||||||
|
if (byte != byte & 0xff) {
|
||||||
|
throw FormatException("Not a byte value", bytes, i);
|
||||||
|
}
|
||||||
|
var character = _characters.codeUnitAt(byte);
|
||||||
|
if (character == 0xFFFD) {
|
||||||
|
throw FormatException("Not defined in this code page", bytes, i);
|
||||||
|
}
|
||||||
|
codeUnits[i] = character;
|
||||||
|
}
|
||||||
|
return String.fromCharCodes(codeUnits);
|
||||||
|
}
|
||||||
|
|
||||||
|
String _convertAllowInvalid(List<int> bytes) {
|
||||||
|
var count = bytes.length;
|
||||||
|
var codeUnits = Uint16List(count);
|
||||||
|
for (var i = 0; i < count; i++) {
|
||||||
|
var byte = bytes[i];
|
||||||
|
int character;
|
||||||
|
if (byte == byte & 0xff) {
|
||||||
|
character = _characters.codeUnitAt(byte);
|
||||||
|
} else {
|
||||||
|
character = 0xFFFD;
|
||||||
|
}
|
||||||
|
codeUnits[i] = character;
|
||||||
|
}
|
||||||
|
return String.fromCharCodes(codeUnits);
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
CodePageEncoder _createEncoder() => CodePageEncoder._bmp(_characters);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Encoder for a code page.
|
||||||
|
///
|
||||||
|
/// Converts a string into bytes where each byte represents that character
|
||||||
|
/// according to the code page definition.
|
||||||
|
class CodePageEncoder extends Converter<String, List<int>> {
|
||||||
|
final Map<int, int> _encoding;
|
||||||
|
|
||||||
|
CodePageEncoder._bmp(String characters)
|
||||||
|
: _encoding = _createBmpEncoding(characters);
|
||||||
|
|
||||||
|
CodePageEncoder._(this._encoding);
|
||||||
|
|
||||||
|
static Map<int, int> _createBmpEncoding(String characters) {
|
||||||
|
var encoding = <int, int>{};
|
||||||
|
for (var i = 0; i < characters.length; i++) {
|
||||||
|
var char = characters.codeUnitAt(i);
|
||||||
|
if (char != 0xFFFD) encoding[characters.codeUnitAt(i)] = i;
|
||||||
|
}
|
||||||
|
return encoding;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Converts input to the byte encoding in this code page.
|
||||||
|
///
|
||||||
|
/// If [invalidCharacter] is supplied, it must be a byte value
|
||||||
|
/// (in the range 0..255).
|
||||||
|
///
|
||||||
|
/// If [input] contains characters that are not available
|
||||||
|
/// in this code page, they are replaced by the [invalidCharacter] byte,
|
||||||
|
/// and then [invalidCharacter] must have been supplied.
|
||||||
|
@override
|
||||||
|
Uint8List convert(String input, {int? invalidCharacter}) {
|
||||||
|
if (invalidCharacter != null) {
|
||||||
|
RangeError.checkValueInInterval(
|
||||||
|
invalidCharacter, 0, 255, "invalidCharacter");
|
||||||
|
}
|
||||||
|
var count = input.length;
|
||||||
|
var result = Uint8List(count);
|
||||||
|
var j = 0;
|
||||||
|
for (var i = 0; i < count; i++) {
|
||||||
|
var char = input.codeUnitAt(i);
|
||||||
|
var byte = _encoding[char];
|
||||||
|
nullCheck:
|
||||||
|
if (byte == null) {
|
||||||
|
// Check for surrogate.
|
||||||
|
var offset = i;
|
||||||
|
if (char & 0xFC00 == 0xD800 && i + 1 < count) {
|
||||||
|
var next = input.codeUnitAt(i + 1);
|
||||||
|
if ((next & 0xFC00) == 0xDC00) {
|
||||||
|
i = i + 1;
|
||||||
|
char = 0x10000 + ((char & 0x3ff) << 10) + (next & 0x3ff);
|
||||||
|
byte = _encoding[char];
|
||||||
|
if (byte != null) break nullCheck;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
byte = invalidCharacter ??
|
||||||
|
(throw FormatException(
|
||||||
|
"Not a character in this code page", input, offset));
|
||||||
|
}
|
||||||
|
result[j++] = byte;
|
||||||
|
}
|
||||||
|
return Uint8List.sublistView(result, 0, j);
|
||||||
|
}
|
||||||
|
}
|
||||||
78
lib/charset/euc_jp.dart
Normal file
78
lib/charset/euc_jp.dart
Normal file
|
|
@ -0,0 +1,78 @@
|
||||||
|
import 'dart:convert';
|
||||||
|
import 'dart:math';
|
||||||
|
import 'euc_jp_table.dart';
|
||||||
|
|
||||||
|
const eucJp = EucJPCodec();
|
||||||
|
|
||||||
|
class EucJPDecoder extends Converter<List<int>, String> {
|
||||||
|
final bool _allowMalformed;
|
||||||
|
const EucJPDecoder([this._allowMalformed = false]);
|
||||||
|
@override
|
||||||
|
convert(input) {
|
||||||
|
List<int> result = [];
|
||||||
|
for (int i = 0; i < input.length; i++) {
|
||||||
|
final c1 = input[i];
|
||||||
|
final List<int>? codes;
|
||||||
|
if (c1 <= 0x7E) {
|
||||||
|
// ASCII Compatible
|
||||||
|
codes = EUC_TABLE[c1];
|
||||||
|
} else if (c1 == 0x8e) {
|
||||||
|
// Hiragana
|
||||||
|
final c2 = input[++i];
|
||||||
|
codes = EUC_TABLE[(c1 << 8) + c2];
|
||||||
|
} else if (c1 == 0x8f) {
|
||||||
|
// JIS X 0212
|
||||||
|
final c2 = input[++i];
|
||||||
|
final c3 = input[++i];
|
||||||
|
codes = EUC_TABLE[(c1 << 16) + (c2 << 8) + c3];
|
||||||
|
} else {
|
||||||
|
// JIS X 0208
|
||||||
|
final c2 = input[++i];
|
||||||
|
codes = EUC_TABLE[(c1 << 8) + c2];
|
||||||
|
}
|
||||||
|
if (codes == null) {
|
||||||
|
if (_allowMalformed) {
|
||||||
|
throw FormatException('Unfinished Euc-JP octet sequence', input, i);
|
||||||
|
}
|
||||||
|
result.add(unicodeReplacementCharacterRune);
|
||||||
|
} else {
|
||||||
|
result.addAll(codes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return utf8.decode(result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class EucJPEncoder extends Converter<String, List<int>> {
|
||||||
|
const EucJPEncoder();
|
||||||
|
@override
|
||||||
|
List<int> convert(String input) {
|
||||||
|
List<int> result = [];
|
||||||
|
for (int i = 0; i < input.length; i++) {
|
||||||
|
var bytes = utf8.encode(input[i]);
|
||||||
|
var value = 0;
|
||||||
|
|
||||||
|
for (var i = 0, length = bytes.length; i < length; i++) {
|
||||||
|
value += bytes[i] * (pow(256, (bytes.length - i - 1)) as int);
|
||||||
|
}
|
||||||
|
|
||||||
|
result.addAll(UTF_TABLE[value] ?? []);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class EucJPCodec extends Encoding {
|
||||||
|
final bool _allowMalformed;
|
||||||
|
const EucJPCodec([this._allowMalformed = false]);
|
||||||
|
|
||||||
|
@override
|
||||||
|
Converter<List<int>, String> get decoder =>
|
||||||
|
_allowMalformed ? const EucJPDecoder(true) : const EucJPDecoder();
|
||||||
|
|
||||||
|
@override
|
||||||
|
Converter<String, List<int>> get encoder => const EucJPEncoder();
|
||||||
|
|
||||||
|
@override
|
||||||
|
String get name => 'euc-jp';
|
||||||
|
}
|
||||||
26278
lib/charset/euc_jp_table.dart
Normal file
26278
lib/charset/euc_jp_table.dart
Normal file
File diff suppressed because it is too large
Load diff
90
lib/charset/euc_kr.dart
Normal file
90
lib/charset/euc_kr.dart
Normal file
|
|
@ -0,0 +1,90 @@
|
||||||
|
import 'dart:convert';
|
||||||
|
|
||||||
|
import 'euc_kr_table.dart';
|
||||||
|
|
||||||
|
const eucKr = EucKRCodec(true);
|
||||||
|
|
||||||
|
class EucKRCodec extends Encoding {
|
||||||
|
final bool _allowInvalid;
|
||||||
|
|
||||||
|
const EucKRCodec([this._allowInvalid = false]) : super();
|
||||||
|
|
||||||
|
@override
|
||||||
|
String get name => "euc-kr";
|
||||||
|
|
||||||
|
@override
|
||||||
|
EucKREncoder get encoder => const EucKREncoder();
|
||||||
|
|
||||||
|
@override
|
||||||
|
EucKRDecoder get decoder =>
|
||||||
|
_allowInvalid ? const EucKRDecoder(true) : const EucKRDecoder();
|
||||||
|
}
|
||||||
|
|
||||||
|
class EucKREncoder extends Converter<String, List<int>> {
|
||||||
|
const EucKREncoder();
|
||||||
|
|
||||||
|
@override
|
||||||
|
List<int> convert(String input) {
|
||||||
|
List<int> bits = [];
|
||||||
|
for (var i in input.codeUnits) {
|
||||||
|
if (i < 0x80) {
|
||||||
|
bits.add(i);
|
||||||
|
} else {
|
||||||
|
int code = utf8ToEucKr[i] ?? 0;
|
||||||
|
if (code > 0) {
|
||||||
|
// bits.add(code ~/ 190 + 0x81);
|
||||||
|
// bits.add(code % 190 + 0x41);
|
||||||
|
bits.add(code >> 8);
|
||||||
|
bits.add(code & 0xff);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return bits;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class EucKRDecoder extends Converter<List<int>, String> {
|
||||||
|
final bool _allowInvalid;
|
||||||
|
const EucKRDecoder([this._allowInvalid = false]);
|
||||||
|
|
||||||
|
@override
|
||||||
|
String convert(List<int> input) {
|
||||||
|
int leadPointer = 0;
|
||||||
|
StringBuffer sb = StringBuffer();
|
||||||
|
for (int i = 0; i < input.length; i++) {
|
||||||
|
int pointer = input[i];
|
||||||
|
if (leadPointer != 0) {
|
||||||
|
if (pointer >= 0x41 && pointer <= 0xfe) {
|
||||||
|
//int code = (leadPointer - 0x81) * 190 + (pointer - 0x41);
|
||||||
|
int code = (leadPointer << 8) + pointer;
|
||||||
|
final charCode = code < 0x80 ? code : utf8ToEucKr[code];
|
||||||
|
if (charCode == null && !_allowInvalid) {
|
||||||
|
throw FormatException('Unfinished Euc-KR octet sequence', input, i);
|
||||||
|
} else {
|
||||||
|
sb.writeCharCode(charCode ?? unicodeReplacementCharacterRune);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
leadPointer = 0;
|
||||||
|
} else if (pointer < 0x80) {
|
||||||
|
sb.writeCharCode(pointer);
|
||||||
|
} else if (pointer >= 0x81 && pointer <= 0xfe) {
|
||||||
|
leadPointer = pointer;
|
||||||
|
} else {
|
||||||
|
if (!_allowInvalid) {
|
||||||
|
throw FormatException('Unfinished Euc-KR octet sequence', input, i);
|
||||||
|
}
|
||||||
|
sb.writeCharCode(unicodeReplacementCharacterRune);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (leadPointer != 0) {
|
||||||
|
if (!_allowInvalid) {
|
||||||
|
throw FormatException(
|
||||||
|
'Unfinished Euc-KR octet sequence',
|
||||||
|
input,
|
||||||
|
leadPointer,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
}
|
||||||
34652
lib/charset/euc_kr_table.dart
Normal file
34652
lib/charset/euc_kr_table.dart
Normal file
File diff suppressed because it is too large
Load diff
436
lib/charset/gbk.dart
Normal file
436
lib/charset/gbk.dart
Normal file
|
|
@ -0,0 +1,436 @@
|
||||||
|
// ignore_for_file: constant_identifier_names, duplicate_ignore
|
||||||
|
|
||||||
|
import 'dart:convert';
|
||||||
|
import 'dart:typed_data';
|
||||||
|
import 'gbk_encoder_map.dart';
|
||||||
|
import 'gbk_decoder_map.dart';
|
||||||
|
|
||||||
|
/// The GBK Replacement character `U+E7B3` (). GBK 0xA7F6
|
||||||
|
const int replacementCharacterUnicode = 0xE7B3;
|
||||||
|
const int replacementCharacterGBK = 0xA7F6;
|
||||||
|
|
||||||
|
/// The Unicode Byte Order Marker (BOM) character `U+FEFF`.
|
||||||
|
const int unicodeBomCharacterRune = 0xFEFF;
|
||||||
|
|
||||||
|
/// An instance of the default implementation of the [GbkCodec].
|
||||||
|
///
|
||||||
|
/// This instance provides a convenient access to the most common GBK
|
||||||
|
/// use cases.
|
||||||
|
///
|
||||||
|
/// Examples:
|
||||||
|
///
|
||||||
|
/// List<int> encoded = gbk.encode("¤§¨°±");
|
||||||
|
/// String decoded = gbk.decode([0xA1,0xE8,0xA1,0xEC,
|
||||||
|
/// 0xA1,0xA7,0xA1,0xE3,0xA1,0xC0]);
|
||||||
|
const GbkCodec gbk = GbkCodec();
|
||||||
|
|
||||||
|
/// A [GbkCodec] encodes strings to GBK code units (bytes) and decodes
|
||||||
|
/// GBK code units to strings.
|
||||||
|
class GbkCodec extends Encoding {
|
||||||
|
final bool _allowMalformed;
|
||||||
|
|
||||||
|
/// Instantiates a new [GbkCodec].
|
||||||
|
///
|
||||||
|
/// The optional [allowMalformed] argument defines how [decoder] (and [decode])
|
||||||
|
/// deal with invalid or unterminated character sequences.
|
||||||
|
///
|
||||||
|
/// If it is `true` (and not overridden at the method invocation) [decode] and
|
||||||
|
/// the [decoder] replace invalid (or unterminated) octet
|
||||||
|
/// sequences with the Unicode Replacement character `U+FFFD` (<EFBFBD>). Otherwise
|
||||||
|
/// they throw a [FormatException].
|
||||||
|
const GbkCodec({bool allowMalformed = false})
|
||||||
|
: _allowMalformed = allowMalformed;
|
||||||
|
|
||||||
|
/// The name of this codec, 'gbk'.
|
||||||
|
@override
|
||||||
|
String get name => 'gbk';
|
||||||
|
|
||||||
|
/// Decodes the UTF-8 [codeUnits] (a list of unsigned 8-bit integers) to the
|
||||||
|
/// corresponding string.
|
||||||
|
///
|
||||||
|
/// If the [codeUnits] start with the encoding of a
|
||||||
|
/// [unicodeBomCharacterRune], that character is discarded.
|
||||||
|
///
|
||||||
|
/// If [allowMalformed] is `true` the decoder replaces invalid (or
|
||||||
|
/// unterminated) character sequences with the Unicode Replacement character
|
||||||
|
/// `U+FFFD` (<EFBFBD>). Otherwise it throws a [FormatException].
|
||||||
|
///
|
||||||
|
/// If [allowMalformed] is not given, it defaults to the `allowMalformed` that
|
||||||
|
/// was used to instantiate `this`.
|
||||||
|
@override
|
||||||
|
String decode(List<int> codeUnits, {bool? allowMalformed}) {
|
||||||
|
allowMalformed ??= _allowMalformed;
|
||||||
|
return GbkDecoder(allowMalformed: allowMalformed).convert(codeUnits);
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
GbkDecoder get decoder {
|
||||||
|
return GbkDecoder(allowMalformed: _allowMalformed);
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
GbkEncoder get encoder => const GbkEncoder();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// This class converts strings to their GBK code units (a list of
|
||||||
|
/// unsigned 8-bit integers).
|
||||||
|
class GbkEncoder extends Converter<String, List<int>> {
|
||||||
|
const GbkEncoder();
|
||||||
|
|
||||||
|
/// Converts [string] to its GBK code units (a list of
|
||||||
|
/// unsigned 8-bit integers).
|
||||||
|
///
|
||||||
|
/// If [start] and [end] are provided, only the substring
|
||||||
|
/// `string.substring(start, end)` is converted.
|
||||||
|
@override
|
||||||
|
Uint8List convert(String string, [int start = 0, int? end]) {
|
||||||
|
var stringLength = string.length;
|
||||||
|
end = RangeError.checkValidRange(start, end, stringLength);
|
||||||
|
var length = end - start;
|
||||||
|
if (length == 0) return Uint8List(0);
|
||||||
|
|
||||||
|
var encoder = _GbkStreamEncoder.withBufferSize(stringLength * 2);
|
||||||
|
var ending = encoder.encode(string, start, end);
|
||||||
|
return encoder._buffer.sublist(0, ending);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Starts a chunked conversion.
|
||||||
|
///
|
||||||
|
/// The converter works more efficiently if the given [sink] is a
|
||||||
|
/// [ByteConversionSink].
|
||||||
|
@override
|
||||||
|
StringConversionSink startChunkedConversion(Sink<List<int>> sink) {
|
||||||
|
return _GbkEncoderSink(
|
||||||
|
(sink is ByteConversionSink) ? sink : ByteConversionSink.from(sink),
|
||||||
|
_GbkStreamEncoder());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// This class encodes Strings to UTF-8 code units (unsigned 8 bit integers).
|
||||||
|
class _GbkStreamEncoder {
|
||||||
|
final Uint8List _buffer;
|
||||||
|
|
||||||
|
static const _DEFAULT_BYTE_BUFFER_SIZE = 1024;
|
||||||
|
|
||||||
|
_GbkStreamEncoder() : this.withBufferSize(_DEFAULT_BYTE_BUFFER_SIZE);
|
||||||
|
|
||||||
|
_GbkStreamEncoder.withBufferSize(int bufferSize)
|
||||||
|
: _buffer = _createBuffer(bufferSize);
|
||||||
|
|
||||||
|
/// Allow an implementation to pick the most efficient way of storing bytes.
|
||||||
|
static Uint8List _createBuffer(int size) => Uint8List(size);
|
||||||
|
|
||||||
|
int encode(String input, int start, int end) {
|
||||||
|
var source = input.codeUnits;
|
||||||
|
var srcIndex = 0;
|
||||||
|
var targetIndex = 0;
|
||||||
|
|
||||||
|
while (srcIndex < source.length) {
|
||||||
|
var codeUnit = source[srcIndex];
|
||||||
|
// ignore non-BMP String character
|
||||||
|
if (_isLeadSurrogate(codeUnit) || _isTailSurrogate(codeUnit)) {
|
||||||
|
_buffer[targetIndex++] = replacementCharacterGBK;
|
||||||
|
srcIndex++;
|
||||||
|
srcIndex++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_isAscii(codeUnit)) {
|
||||||
|
_buffer[targetIndex++] = codeUnit;
|
||||||
|
srcIndex++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var gbkCode = utf16ToGBKMap[codeUnit];
|
||||||
|
if (gbkCode != null) {
|
||||||
|
_buffer[targetIndex++] = (gbkCode >> 8) & 0xff;
|
||||||
|
_buffer[targetIndex++] = gbkCode & 0xff;
|
||||||
|
} else {
|
||||||
|
// unknown GBK code;
|
||||||
|
_buffer[targetIndex++] = (replacementCharacterGBK >> 8) & 0xff;
|
||||||
|
|
||||||
|
_buffer[targetIndex++] = replacementCharacterGBK & 0xff;
|
||||||
|
}
|
||||||
|
srcIndex++;
|
||||||
|
}
|
||||||
|
return targetIndex;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// This class encodes chunked strings to GBK code units (unsigned 8-bit
|
||||||
|
/// integers).
|
||||||
|
/// stateless, String input, 2Bytes GBK output.
|
||||||
|
class _GbkEncoderSink with StringConversionSinkMixin {
|
||||||
|
final ByteConversionSink _sink;
|
||||||
|
final _GbkStreamEncoder _encoder;
|
||||||
|
|
||||||
|
_GbkEncoderSink(this._sink, this._encoder);
|
||||||
|
|
||||||
|
@override
|
||||||
|
void close() {
|
||||||
|
_sink.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
void addSlice(String input, int start, int end, bool isLast) {
|
||||||
|
var index = _encoder.encode(input, start, end);
|
||||||
|
_sink.addSlice(_encoder._buffer, 0, index, isLast);
|
||||||
|
|
||||||
|
if (isLast) close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// This class converts GBK code units (lists of unsigned 8-bit integers)
|
||||||
|
/// to a string.
|
||||||
|
class GbkDecoder extends Converter<List<int>, String> {
|
||||||
|
final bool _allowMalformed;
|
||||||
|
|
||||||
|
/// Instantiates a new [GbkDecoder].
|
||||||
|
///
|
||||||
|
/// The optional [allowMalformed] argument defines how [convert] deals
|
||||||
|
/// with invalid or unterminated character sequences.
|
||||||
|
///
|
||||||
|
/// If it is `true` [convert] replaces invalid (or unterminated) character
|
||||||
|
/// sequences with the Unicode Replacement character `U+FFFD` (<EFBFBD>). Otherwise
|
||||||
|
/// it throws a [FormatException].
|
||||||
|
const GbkDecoder({bool allowMalformed = false})
|
||||||
|
: _allowMalformed = allowMalformed;
|
||||||
|
|
||||||
|
/// Converts the GBK [codeUnits] (a list of unsigned 8-bit integers) to the
|
||||||
|
/// corresponding string.
|
||||||
|
///
|
||||||
|
/// Uses the code units from [start] to, but no including, [end].
|
||||||
|
/// If [end] is omitted, it defaults to `codeUnits.length`.
|
||||||
|
///
|
||||||
|
/// If the [codeUnits] start with the encoding of a
|
||||||
|
/// [unicodeBomCharacterRune], that character is discarded.
|
||||||
|
@override
|
||||||
|
String convert(List<int> codeUnits, [int start = 0, int? end]) {
|
||||||
|
var length = codeUnits.length;
|
||||||
|
end = RangeError.checkValidRange(start, end, length);
|
||||||
|
|
||||||
|
// Fast case for ASCII strings avoids StringBuffer / decodeMap.
|
||||||
|
var oneBytes = _scanOneByteCharacters(codeUnits, start, end);
|
||||||
|
StringBuffer? buffer;
|
||||||
|
if (oneBytes > 0) {
|
||||||
|
var firstPart = String.fromCharCodes(codeUnits, start, start + oneBytes);
|
||||||
|
start += oneBytes;
|
||||||
|
if (start == end) {
|
||||||
|
return firstPart;
|
||||||
|
}
|
||||||
|
buffer = StringBuffer(firstPart);
|
||||||
|
}
|
||||||
|
|
||||||
|
buffer ??= StringBuffer();
|
||||||
|
var decoder = _GbkStreamDecoder(buffer, _allowMalformed);
|
||||||
|
decoder.convert(codeUnits, start, end);
|
||||||
|
decoder.flush(codeUnits, end);
|
||||||
|
return buffer.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Starts a chunked conversion.
|
||||||
|
///
|
||||||
|
/// The converter works more efficiently if the given [sink] is a
|
||||||
|
/// [StringConversionSink].
|
||||||
|
@override
|
||||||
|
ByteConversionSink startChunkedConversion(Sink<String> sink) {
|
||||||
|
StringConversionSink stringSink;
|
||||||
|
if (sink is StringConversionSink) {
|
||||||
|
stringSink = sink;
|
||||||
|
} else {
|
||||||
|
stringSink = StringConversionSink.from(sink);
|
||||||
|
}
|
||||||
|
|
||||||
|
return _GbkConversionSink(stringSink, _allowMalformed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decodes GBK code units.
|
||||||
|
///
|
||||||
|
/// Forwards the decoded strings to the given [StringConversionSink].
|
||||||
|
class _GbkConversionSink extends ByteConversionSink {
|
||||||
|
final _GbkStreamDecoder _decoder;
|
||||||
|
final StringConversionSink _chunkedSink;
|
||||||
|
final StringBuffer _buffer;
|
||||||
|
|
||||||
|
_GbkConversionSink(StringConversionSink sink, bool allowMalformed)
|
||||||
|
: this._(sink, StringBuffer(), allowMalformed);
|
||||||
|
|
||||||
|
_GbkConversionSink._(
|
||||||
|
this._chunkedSink, StringBuffer stringBuffer, bool allowMalformed)
|
||||||
|
: _decoder = _GbkStreamDecoder(stringBuffer, allowMalformed),
|
||||||
|
_buffer = stringBuffer;
|
||||||
|
|
||||||
|
@override
|
||||||
|
void close() {
|
||||||
|
_decoder.close();
|
||||||
|
if (_buffer.isNotEmpty) {
|
||||||
|
var accumulated = _buffer.toString();
|
||||||
|
_buffer.clear();
|
||||||
|
_chunkedSink.addSlice(accumulated, 0, accumulated.length, true);
|
||||||
|
} else {
|
||||||
|
_chunkedSink.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
void add(List<int> chunk) {
|
||||||
|
addSlice(chunk, 0, chunk.length, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
void addSlice(List<int> chunk, int startIndex, int endIndex, bool isLast) {
|
||||||
|
_decoder.convert(chunk, startIndex, endIndex);
|
||||||
|
if (_buffer.isNotEmpty) {
|
||||||
|
var accumulated = _buffer.toString();
|
||||||
|
_chunkedSink.addSlice(accumulated, 0, accumulated.length, isLast);
|
||||||
|
_buffer.clear();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (isLast) close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decodes GBK.
|
||||||
|
///
|
||||||
|
/// The decoder handles chunked input.
|
||||||
|
///
|
||||||
|
/// init() -> convert -> convert -> ... -> close() -> flush()
|
||||||
|
///
|
||||||
|
class _GbkStreamDecoder {
|
||||||
|
// throw exception or not
|
||||||
|
final bool _allowMalformed;
|
||||||
|
|
||||||
|
// output of the Decoder
|
||||||
|
final StringSink _stringSink;
|
||||||
|
|
||||||
|
// GBK need 2 bytes, if only 1 byte received, store here.
|
||||||
|
var _firstByte = -1;
|
||||||
|
|
||||||
|
bool get hasPartialInput => _firstByte > -1;
|
||||||
|
|
||||||
|
_GbkStreamDecoder(this._stringSink, this._allowMalformed);
|
||||||
|
|
||||||
|
void close() {
|
||||||
|
flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Flushes this decoder as if closed.
|
||||||
|
///
|
||||||
|
/// This method throws if the input was partial and the decoder was
|
||||||
|
/// constructed with `allowMalformed` set to `false`.
|
||||||
|
///
|
||||||
|
/// The [source] and [offset] of the current position may be provided,
|
||||||
|
/// and are included in the exception if one is thrown.
|
||||||
|
void flush([List<int>? source, int? offset]) {
|
||||||
|
if (hasPartialInput) {
|
||||||
|
if (!_allowMalformed) {
|
||||||
|
throw FormatException('Unfinished GBK octet sequence', source, offset);
|
||||||
|
}
|
||||||
|
_stringSink.writeCharCode(unicodeReplacementCharacterRune);
|
||||||
|
_firstByte = -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void convert(List<int> codeUnits, int startIndex, int endIndex) {
|
||||||
|
var begin = startIndex;
|
||||||
|
// if _firstByte > 0, we need to finish last time's job first.
|
||||||
|
if (hasPartialInput) {
|
||||||
|
var code = ((_firstByte) << 8) + (codeUnits[0] & 0xff);
|
||||||
|
var char = gbkToUtf16Map[code];
|
||||||
|
if (char == null && !_allowMalformed) {
|
||||||
|
throw FormatException(
|
||||||
|
'Bad GBK encoding 0x${code.toRadixString(16)}', code);
|
||||||
|
}
|
||||||
|
if (char != null) {
|
||||||
|
_stringSink.write(String.fromCharCode(char));
|
||||||
|
} else {
|
||||||
|
_stringSink.write(String.fromCharCode(unicodeReplacementCharacterRune));
|
||||||
|
}
|
||||||
|
_firstByte = -1;
|
||||||
|
begin += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// handle new incoming data.
|
||||||
|
for (var index = begin; index < endIndex; index++) {
|
||||||
|
var code = codeUnits[index];
|
||||||
|
if (_isAscii(codeUnits[index])) {
|
||||||
|
_stringSink.writeCharCode(code);
|
||||||
|
} else {
|
||||||
|
index++;
|
||||||
|
//GBK need 2 bytes. wait for more to come.
|
||||||
|
if (index == endIndex) {
|
||||||
|
_firstByte = code;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
code = ((code) << 8) + (codeUnits[index] & 0xff);
|
||||||
|
if (code == unicodeBomCharacterRune) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
var char = gbkToUtf16Map[code];
|
||||||
|
if (char == null && !_allowMalformed) {
|
||||||
|
throw FormatException(
|
||||||
|
'Bad GBK encoding 0x${code.toRadixString(16)}', code);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (char != null) {
|
||||||
|
_stringSink.write(String.fromCharCode(char));
|
||||||
|
} else {
|
||||||
|
_stringSink
|
||||||
|
.write(String.fromCharCode(unicodeReplacementCharacterRune));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
///
|
||||||
|
/// GBK的编码范围
|
||||||
|
/// 范围 第1字节 第2字节 编码数 字数
|
||||||
|
/// 水准GBK/1 A1–A9 A1–FE 846 717
|
||||||
|
/// 水准GBK/2 B0–F7 A1–FE 6,768 6,763
|
||||||
|
/// 水准GBK/3 81–A0 40–FE (7F除外) 6,080 6,080
|
||||||
|
/// 水准GBK/4 AA–FE 40–A0 (7F除外) 8,160 8,160
|
||||||
|
/// 水准GBK/5 A8–A9 40–A0 (7F除外) 192 166
|
||||||
|
/// 用户定义 AA–AF A1–FE 564
|
||||||
|
/// 用户定义 F8–FE A1–FE 658
|
||||||
|
/// 用户定义 A1–A7 40–A0 (7F除外) 672
|
||||||
|
/// 合计： 23,940 21,886
|
||||||
|
///
|
||||||
|
|
||||||
|
int _scanOneByteCharacters(List<int> units, int from, int endIndex) {
|
||||||
|
final to = endIndex;
|
||||||
|
for (var i = from; i < to; i++) {
|
||||||
|
final unit = units[i];
|
||||||
|
if ((unit & _ONE_BYTE_LIMIT) != unit) return i - from;
|
||||||
|
}
|
||||||
|
return to - from;
|
||||||
|
}
|
||||||
|
|
||||||
|
///
|
||||||
|
/// For a character outside the Basic Multilingual Plane (plane 0) that is
|
||||||
|
/// composed of a surrogate pair, [runes] combines the pair and returns a
|
||||||
|
/// single integer. For example, the Unicode character for a
|
||||||
|
/// musical G-clef ('𝄞') with rune value 0x1D11E consists of a UTF-16 surrogate
|
||||||
|
/// pair: `0xD834` and `0xDD1E`. Using [codeUnits] returns the surrogate pair,
|
||||||
|
/// and using `runes` returns their combined value:
|
||||||
|
///
|
||||||
|
/// var clef = '\u{1D11E}';
|
||||||
|
/// clef.codeUnits; // [0xD834, 0xDD1E]
|
||||||
|
/// clef.runes.toList(); // [0x1D11E]
|
||||||
|
///
|
||||||
|
/// UTF-16 constants.
|
||||||
|
/// https://zh.wikipedia.org/wiki/UTF-16
|
||||||
|
const int _SURROGATE_TAG_MASK = 0xFC00;
|
||||||
|
//const int _SURROGATE_VALUE_MASK = 0x3FF;
|
||||||
|
const int _LEAD_SURROGATE_MIN = 0xD800;
|
||||||
|
const int _TAIL_SURROGATE_MIN = 0xDC00;
|
||||||
|
|
||||||
|
bool _isLeadSurrogate(int codeUnit) =>
|
||||||
|
(codeUnit & _SURROGATE_TAG_MASK) == _LEAD_SURROGATE_MIN;
|
||||||
|
|
||||||
|
bool _isTailSurrogate(int codeUnit) =>
|
||||||
|
(codeUnit & _SURROGATE_TAG_MASK) == _TAIL_SURROGATE_MIN;
|
||||||
|
|
||||||
|
const int _ONE_BYTE_LIMIT = 0x7f; // 7 bits
|
||||||
|
bool _isAscii(int codeUnit) => (codeUnit <= _ONE_BYTE_LIMIT);
|
||||||
23943
lib/charset/gbk_decoder_map.dart
Normal file
23943
lib/charset/gbk_decoder_map.dart
Normal file
File diff suppressed because it is too large
Load diff
24009
lib/charset/gbk_encoder_map.dart
Normal file
24009
lib/charset/gbk_encoder_map.dart
Normal file
File diff suppressed because it is too large
Load diff
72
lib/charset/map_code_page.dart
Normal file
72
lib/charset/map_code_page.dart
Normal file
|
|
@ -0,0 +1,72 @@
|
||||||
|
import 'dart:convert';
|
||||||
|
|
||||||
|
abstract class MappedPageCodec extends Encoding {
|
||||||
|
final MappedPageEncoder _encoder;
|
||||||
|
final MappedPageDecoder _decoder;
|
||||||
|
|
||||||
|
const MappedPageCodec(this._encoder, this._decoder) : super();
|
||||||
|
|
||||||
|
@override
|
||||||
|
MappedPageEncoder get encoder => _encoder;
|
||||||
|
|
||||||
|
@override
|
||||||
|
MappedPageDecoder get decoder => _decoder;
|
||||||
|
}
|
||||||
|
|
||||||
|
class MappedPageEncoder extends Converter<String, List<int>> {
|
||||||
|
final Map<int, int> coderMap;
|
||||||
|
const MappedPageEncoder(this.coderMap);
|
||||||
|
|
||||||
|
@override
|
||||||
|
List<int> convert(String input) {
|
||||||
|
List<int> bits = [];
|
||||||
|
for (var i in input.codeUnits) {
|
||||||
|
if (i < 0x80) {
|
||||||
|
bits.add(i);
|
||||||
|
} else {
|
||||||
|
int code = coderMap[i] ?? 0;
|
||||||
|
if (code > 0) {
|
||||||
|
// bits.add(code ~/ 190 + 0x81);
|
||||||
|
// bits.add(code % 190 + 0x41);
|
||||||
|
bits.add(code >> 8);
|
||||||
|
bits.add(code & 0xff);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return bits;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class MappedPageDecoder extends Converter<List<int>, String> {
|
||||||
|
final bool _allowInvalid;
|
||||||
|
final Map<int, int> coderMap;
|
||||||
|
const MappedPageDecoder(this.coderMap, [this._allowInvalid = false]);
|
||||||
|
|
||||||
|
@override
|
||||||
|
String convert(List<int> input) {
|
||||||
|
int leadPointer = 0;
|
||||||
|
StringBuffer sb = StringBuffer();
|
||||||
|
for (int i = 0; i < input.length; i++) {
|
||||||
|
int pointer = input[i];
|
||||||
|
if (leadPointer != 0) {
|
||||||
|
if (pointer >= 0x41 && pointer <= 0xfe) {
|
||||||
|
//int code = (leadPointer - 0x81) * 190 + (pointer - 0x41);
|
||||||
|
int code = leadPointer << 8 + pointer;
|
||||||
|
|
||||||
|
sb.writeCharCode(code < 0x80 ? code : (coderMap[code] ?? 0));
|
||||||
|
}
|
||||||
|
leadPointer = 0;
|
||||||
|
} else if (pointer < 0x80) {
|
||||||
|
sb.writeCharCode(pointer);
|
||||||
|
} else if (pointer >= 0x81 && pointer <= 0xfe) {
|
||||||
|
leadPointer = pointer;
|
||||||
|
} else {
|
||||||
|
if (!_allowInvalid) throw Exception('');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (leadPointer != 0) {
|
||||||
|
if (!_allowInvalid) throw Exception('');
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
}
|
||||||
79
lib/charset/shift_jis.dart
Normal file
79
lib/charset/shift_jis.dart
Normal file
|
|
@ -0,0 +1,79 @@
|
||||||
|
import 'dart:convert';
|
||||||
|
import 'dart:math';
|
||||||
|
import 'shift_jis_table.dart';
|
||||||
|
|
||||||
|
class ShiftJISDecoder extends Converter<List<int>, String> {
|
||||||
|
final bool _allowMalformed;
|
||||||
|
const ShiftJISDecoder({bool allowMalformed = false})
|
||||||
|
: _allowMalformed = allowMalformed;
|
||||||
|
@override
|
||||||
|
convert(input) {
|
||||||
|
List<int> result = [];
|
||||||
|
for (int i = 0; i < input.length; i++) {
|
||||||
|
final c1 = input[i];
|
||||||
|
List<int>? c2;
|
||||||
|
if (c1 <= 0x7F) {
|
||||||
|
// ASCII Compatible (partially)
|
||||||
|
c2 = JIS_TABLE[c1];
|
||||||
|
} else if (c1 >= 0xa1 && c1 <= 0xdf) {
|
||||||
|
// Half-width Hiragana
|
||||||
|
c2 = JIS_TABLE[c1];
|
||||||
|
} else if (c1 >= 0x81 && c1 <= 0x9f) {
|
||||||
|
// JIS X 0208
|
||||||
|
c2 = JIS_TABLE[(c1 << 8) + input[++i]];
|
||||||
|
} else if (c1 >= 0xe0 && c1 <= 0xef) {
|
||||||
|
// JIS X 0208
|
||||||
|
c2 = JIS_TABLE[(c1 << 8) + input[++i]];
|
||||||
|
} else {
|
||||||
|
// Unknown
|
||||||
|
}
|
||||||
|
if (c2 != null) {
|
||||||
|
result.addAll(c2);
|
||||||
|
} else if (_allowMalformed) {
|
||||||
|
result.add(unicodeReplacementCharacterRune);
|
||||||
|
} else {
|
||||||
|
throw FormatException('Unfinished Shift-JIS octet sequence', input, i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return utf8.decode(result, allowMalformed: _allowMalformed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class ShiftJISEncoder extends Converter<String, List<int>> {
|
||||||
|
const ShiftJISEncoder();
|
||||||
|
|
||||||
|
@override
|
||||||
|
List<int> convert(String input) {
|
||||||
|
List<int> result = [];
|
||||||
|
for (int i = 0; i < input.length; i++) {
|
||||||
|
var bytes = utf8.encode(input[i]);
|
||||||
|
var value = 0;
|
||||||
|
|
||||||
|
for (var i = 0, length = bytes.length; i < length; i++) {
|
||||||
|
value += bytes[i] * (pow(256, (bytes.length - i - 1)) as int);
|
||||||
|
}
|
||||||
|
|
||||||
|
result.addAll(UTF_TABLE[value] ?? [0xFFFD]);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class ShiftJISCodec extends Encoding {
|
||||||
|
final bool _allowMalformed;
|
||||||
|
const ShiftJISCodec({bool allowMalformed = false})
|
||||||
|
: _allowMalformed = allowMalformed;
|
||||||
|
|
||||||
|
@override
|
||||||
|
Converter<List<int>, String> get decoder => _allowMalformed
|
||||||
|
? const ShiftJISDecoder(allowMalformed: true)
|
||||||
|
: const ShiftJISDecoder(allowMalformed: false);
|
||||||
|
|
||||||
|
@override
|
||||||
|
Converter<String, List<int>> get encoder => const ShiftJISEncoder();
|
||||||
|
|
||||||
|
@override
|
||||||
|
String get name => 'shift-jis';
|
||||||
|
}
|
||||||
|
|
||||||
|
const shiftJis = ShiftJISCodec();
|
||||||
14230
lib/charset/shift_jis_table.dart
Normal file
14230
lib/charset/shift_jis_table.dart
Normal file
File diff suppressed because it is too large
Load diff
17
lib/charset/utf/constants.dart
Normal file
17
lib/charset/utf/constants.dart
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
/// Invalid codepoints or encodings may be substituted with the value U+fffd.
|
||||||
|
const int unicodeReplacementCharacterCodepoint = 0xfffd;
|
||||||
|
const int unicodeBom = 0xfeff;
|
||||||
|
const int unicodeUtfBomLo = 0xff;
|
||||||
|
const int unicodeUtfBomHi = 0xfe;
|
||||||
|
|
||||||
|
const int unicodeByteZeroMask = 0xff;
|
||||||
|
const int unicodeByteOneMask = 0xff00;
|
||||||
|
const int unicodeValidRangeMax = 0x10ffff;
|
||||||
|
const int unicodePlaneOneMax = 0xffff;
|
||||||
|
const int unicodeUtf16ReservedLo = 0xd800;
|
||||||
|
const int unicodeUtf16ReservedHi = 0xdfff;
|
||||||
|
const int unicodeUtf16Offset = 0x10000;
|
||||||
|
const int unicodeUtf16SurrogateUnit0Base = 0xd800;
|
||||||
|
const int unicodeUtf16SurrogateUnit1Base = 0xdc00;
|
||||||
|
const int unicodeUtf16HiMask = 0xffc00;
|
||||||
|
const int unicodeUtf16LoMask = 0x3ff;
|
||||||
79
lib/charset/utf/list_range.dart
Normal file
79
lib/charset/utf/list_range.dart
Normal file
|
|
@ -0,0 +1,79 @@
|
||||||
|
import 'dart:collection';
|
||||||
|
|
||||||
|
/// _ListRange in an internal type used to create a lightweight Interable on a
|
||||||
|
/// range within a source list. DO NOT MODIFY the underlying list while
|
||||||
|
/// iterating over it. The results of doing so are undefined.
|
||||||
|
// TODO(floitsch): Consider removing the extend and switch to implements since
|
||||||
|
// that's cheaper to allocate.
|
||||||
|
class ListRange extends IterableBase<int?> {
|
||||||
|
final List<int?> _source;
|
||||||
|
final int _offset;
|
||||||
|
final int _length;
|
||||||
|
|
||||||
|
ListRange(List<int?> source, [int offset = 0, int? length])
|
||||||
|
: _source = source,
|
||||||
|
_offset = offset,
|
||||||
|
_length = (length ?? source.length - offset) {
|
||||||
|
if (_offset < 0 || _offset > _source.length) {
|
||||||
|
throw RangeError.value(_offset);
|
||||||
|
}
|
||||||
|
if (_length < 0) {
|
||||||
|
throw RangeError.value(_length);
|
||||||
|
}
|
||||||
|
if (_length + _offset > _source.length) {
|
||||||
|
throw RangeError.value(_length + _offset);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
ListRangeIterator get iterator =>
|
||||||
|
_ListRangeIteratorImpl(_source, _offset, _offset + _length);
|
||||||
|
|
||||||
|
@override
|
||||||
|
int get length => _length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The ListRangeIterator provides more capabilities than a standard iterator,
|
||||||
|
/// including the ability to get the current position, count remaining items,
|
||||||
|
/// and move forward/backward within the iterator.
|
||||||
|
abstract class ListRangeIterator implements Iterator<int?> {
|
||||||
|
@override
|
||||||
|
bool moveNext();
|
||||||
|
@override
|
||||||
|
int? get current;
|
||||||
|
int get position;
|
||||||
|
void backup([int? by]);
|
||||||
|
int get remaining;
|
||||||
|
void skip([int? count]);
|
||||||
|
}
|
||||||
|
|
||||||
|
class _ListRangeIteratorImpl implements ListRangeIterator {
|
||||||
|
final List<int?> _source;
|
||||||
|
int _offset;
|
||||||
|
final int _end;
|
||||||
|
|
||||||
|
_ListRangeIteratorImpl(this._source, int offset, this._end)
|
||||||
|
: _offset = offset - 1;
|
||||||
|
|
||||||
|
@override
|
||||||
|
int? get current => _source[_offset];
|
||||||
|
|
||||||
|
@override
|
||||||
|
bool moveNext() => ++_offset < _end;
|
||||||
|
|
||||||
|
@override
|
||||||
|
int get position => _offset;
|
||||||
|
|
||||||
|
@override
|
||||||
|
void backup([int? by = 1]) {
|
||||||
|
_offset -= by!;
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
int get remaining => _end - _offset - 1;
|
||||||
|
|
||||||
|
@override
|
||||||
|
void skip([int? count = 1]) {
|
||||||
|
_offset += count!;
|
||||||
|
}
|
||||||
|
}
|
||||||
288
lib/charset/utf/utf16.dart
Normal file
288
lib/charset/utf/utf16.dart
Normal file
|
|
@ -0,0 +1,288 @@
|
||||||
|
import 'dart:convert';
|
||||||
|
import 'dart:typed_data';
|
||||||
|
|
||||||
|
import 'constants.dart';
|
||||||
|
|
||||||
|
import 'list_range.dart';
|
||||||
|
import 'utils.dart';
|
||||||
|
|
||||||
|
class Utf16Codec extends Encoding {
|
||||||
|
const Utf16Codec();
|
||||||
|
|
||||||
|
@override
|
||||||
|
Converter<List<int>, String> get decoder => const Utf16Decoder();
|
||||||
|
|
||||||
|
@override
|
||||||
|
Converter<String, List<int>> get encoder => const Utf16Encoder();
|
||||||
|
|
||||||
|
@override
|
||||||
|
String get name => 'utf-16';
|
||||||
|
}
|
||||||
|
|
||||||
|
class Utf16Encoder extends Converter<String, List<int>> {
|
||||||
|
const Utf16Encoder();
|
||||||
|
|
||||||
|
@override
|
||||||
|
List<int> convert(String input) => encodeUtf16Be(input, true);
|
||||||
|
|
||||||
|
Uint8List encodeUtf16Be(String str, [bool writeBOM = false]) {
|
||||||
|
var utf16CodeUnits = codepointsToUtf16CodeUnits(str.codeUnits);
|
||||||
|
var encoding = Uint8List(2 * utf16CodeUnits.length + (writeBOM ? 2 : 0));
|
||||||
|
var i = 0;
|
||||||
|
if (writeBOM) {
|
||||||
|
encoding[i++] = unicodeUtfBomHi;
|
||||||
|
encoding[i++] = unicodeUtfBomLo;
|
||||||
|
}
|
||||||
|
for (var unit in utf16CodeUnits) {
|
||||||
|
encoding[i++] = (unit! & unicodeByteOneMask) >> 8;
|
||||||
|
encoding[i++] = unit & unicodeByteZeroMask;
|
||||||
|
}
|
||||||
|
return encoding;
|
||||||
|
}
|
||||||
|
|
||||||
|
Uint8List encodeUtf16Le(String str, [bool writeBOM = false]) {
|
||||||
|
var utf16CodeUnits = codepointsToUtf16CodeUnits(str.codeUnits);
|
||||||
|
var encoding = Uint8List(2 * utf16CodeUnits.length + (writeBOM ? 2 : 0));
|
||||||
|
var i = 0;
|
||||||
|
if (writeBOM) {
|
||||||
|
encoding[i++] = unicodeUtfBomLo;
|
||||||
|
encoding[i++] = unicodeUtfBomHi;
|
||||||
|
}
|
||||||
|
for (var unit in utf16CodeUnits) {
|
||||||
|
encoding[i++] = unit! & unicodeByteZeroMask;
|
||||||
|
encoding[i++] = (unit & unicodeByteOneMask) >> 8;
|
||||||
|
}
|
||||||
|
return encoding;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class Utf16Decoder extends Converter<List<int>, String> {
|
||||||
|
const Utf16Decoder();
|
||||||
|
|
||||||
|
/// Produce a String from a sequence of UTF-16 encoded bytes. This method always
|
||||||
|
/// strips a leading BOM. Set the [replacementCodepoint] to null to throw an
|
||||||
|
/// ArgumentError rather than replace the bad value. The default
|
||||||
|
/// value for the [replacementCodepoint] is U+FFFD.
|
||||||
|
@override
|
||||||
|
String convert(List<int> input,
|
||||||
|
[int start = 0,
|
||||||
|
int? end,
|
||||||
|
int replacementCodepoint = unicodeReplacementCharacterCodepoint]) {
|
||||||
|
var codeunits = (Utf16BytesToCodeUnitsDecoder(input, start,
|
||||||
|
end == null ? input.length : end - start, replacementCodepoint))
|
||||||
|
.decodeRest();
|
||||||
|
return String.fromCharCodes(
|
||||||
|
utf16CodeUnitsToCodepoints(codeunits, 0, null, replacementCodepoint)
|
||||||
|
.whereType<int>());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Produce a String from a sequence of UTF-16BE encoded bytes. This method
|
||||||
|
/// strips a leading BOM by default, but can be overridden by setting the
|
||||||
|
/// optional parameter [stripBom] to false. Set the [replacementCodepoint] to
|
||||||
|
/// null to throw an ArgumentError rather than replace the bad value.
|
||||||
|
/// The default value for the [replacementCodepoint] is U+FFFD.
|
||||||
|
String decodeUtf16Be(List<int> input,
|
||||||
|
[int start = 0,
|
||||||
|
int? end,
|
||||||
|
bool stripBom = true,
|
||||||
|
int replacementCodepoint = unicodeReplacementCharacterCodepoint]) {
|
||||||
|
var codeunits = (Utf16beBytesToCodeUnitsDecoder(
|
||||||
|
input,
|
||||||
|
start,
|
||||||
|
end == null ? input.length : end - start,
|
||||||
|
stripBom,
|
||||||
|
replacementCodepoint))
|
||||||
|
.decodeRest();
|
||||||
|
return String.fromCharCodes(
|
||||||
|
utf16CodeUnitsToCodepoints(codeunits, 0, null, replacementCodepoint)
|
||||||
|
.whereType<int>());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Produce a String from a sequence of UTF-16LE encoded bytes. This method
|
||||||
|
/// strips a leading BOM by default, but can be overridden by setting the
|
||||||
|
/// optional parameter [stripBom] to false. Set the [replacementCodepoint] to
|
||||||
|
/// null to throw an ArgumentError rather than replace the bad value.
|
||||||
|
/// The default value for the [replacementCodepoint] is U+FFFD.
|
||||||
|
String decodeUtf16Le(List<int> bytes,
|
||||||
|
[int offset = 0,
|
||||||
|
int? length,
|
||||||
|
bool stripBom = true,
|
||||||
|
int replacementCodepoint = unicodeReplacementCharacterCodepoint]) {
|
||||||
|
var codeunits = (Utf16leBytesToCodeUnitsDecoder(
|
||||||
|
bytes, offset, length, stripBom, replacementCodepoint))
|
||||||
|
.decodeRest();
|
||||||
|
return String.fromCharCodes(
|
||||||
|
utf16CodeUnitsToCodepoints(codeunits, 0, null, replacementCodepoint)
|
||||||
|
.whereType<int>());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const Utf16Codec utf16 = Utf16Codec();
|
||||||
|
|
||||||
|
/// Identifies whether a List of bytes starts (based on offset) with a
|
||||||
|
/// byte-order marker (BOM).
|
||||||
|
bool hasUtf16Bom(List<int> utf32EncodedBytes, [int offset = 0, int? length]) {
|
||||||
|
return hasUtf16BeBom(utf32EncodedBytes, offset, length) ||
|
||||||
|
hasUtf16LeBom(utf32EncodedBytes, offset, length);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Identifies whether a List of bytes starts (based on offset) with a
|
||||||
|
/// big-endian byte-order marker (BOM).
|
||||||
|
bool hasUtf16BeBom(List<int> utf16EncodedBytes, [int offset = 0, int? length]) {
|
||||||
|
var end = length != null ? offset + length : utf16EncodedBytes.length;
|
||||||
|
return (offset + 2) <= end &&
|
||||||
|
utf16EncodedBytes[offset] == unicodeUtfBomHi &&
|
||||||
|
utf16EncodedBytes[offset + 1] == unicodeUtfBomLo;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Identifies whether a List of bytes starts (based on offset) with a
|
||||||
|
/// little-endian byte-order marker (BOM).
|
||||||
|
bool hasUtf16LeBom(List<int> utf16EncodedBytes, [int offset = 0, int? length]) {
|
||||||
|
var end = length != null ? offset + length : utf16EncodedBytes.length;
|
||||||
|
return (offset + 2) <= end &&
|
||||||
|
utf16EncodedBytes[offset] == unicodeUtfBomLo &&
|
||||||
|
utf16EncodedBytes[offset + 1] == unicodeUtfBomHi;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert UTF-16 encoded bytes to UTF-16 code units by grouping 1-2 bytes
|
||||||
|
/// to produce the code unit (0-(2^16)-1). Relies on BOM to determine
|
||||||
|
/// endian-ness, and defaults to BE.
|
||||||
|
abstract class Utf16BytesToCodeUnitsDecoder implements ListRangeIterator {
|
||||||
|
// TODO(kevmoo): should this field be private?
|
||||||
|
final ListRangeIterator utf16EncodedBytesIterator;
|
||||||
|
final int? replacementCodepoint;
|
||||||
|
int? _current;
|
||||||
|
|
||||||
|
Utf16BytesToCodeUnitsDecoder._fromListRangeIterator(
|
||||||
|
this.utf16EncodedBytesIterator, this.replacementCodepoint);
|
||||||
|
|
||||||
|
factory Utf16BytesToCodeUnitsDecoder(List<int> utf16EncodedBytes,
|
||||||
|
[int offset = 0,
|
||||||
|
int? length,
|
||||||
|
int replacementCodepoint = unicodeReplacementCharacterCodepoint]) {
|
||||||
|
length ??= utf16EncodedBytes.length - offset;
|
||||||
|
if (hasUtf16BeBom(utf16EncodedBytes, offset, length)) {
|
||||||
|
return Utf16beBytesToCodeUnitsDecoder(utf16EncodedBytes, offset + 2,
|
||||||
|
length - 2, false, replacementCodepoint);
|
||||||
|
} else if (hasUtf16LeBom(utf16EncodedBytes, offset, length)) {
|
||||||
|
return Utf16leBytesToCodeUnitsDecoder(utf16EncodedBytes, offset + 2,
|
||||||
|
length - 2, false, replacementCodepoint);
|
||||||
|
} else {
|
||||||
|
return Utf16beBytesToCodeUnitsDecoder(
|
||||||
|
utf16EncodedBytes, offset, length, false, replacementCodepoint);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Provides a fast way to decode the rest of the source bytes in a single
|
||||||
|
/// call. This method trades memory for improved speed in that it potentially
|
||||||
|
/// over-allocates the List containing results.
|
||||||
|
List<int?> decodeRest() {
|
||||||
|
var codeunits = <int?>[]..length = remaining;
|
||||||
|
var i = 0;
|
||||||
|
while (moveNext()) {
|
||||||
|
codeunits[i++] = current;
|
||||||
|
}
|
||||||
|
if (i == codeunits.length) {
|
||||||
|
return codeunits;
|
||||||
|
} else {
|
||||||
|
var truncCodeunits = <int?>[]..length = i;
|
||||||
|
truncCodeunits.setRange(0, i, codeunits);
|
||||||
|
return truncCodeunits;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
int? get current => _current;
|
||||||
|
|
||||||
|
@override
|
||||||
|
bool moveNext() {
|
||||||
|
_current = null;
|
||||||
|
var remaining = utf16EncodedBytesIterator.remaining;
|
||||||
|
if (remaining == 0) {
|
||||||
|
_current = null;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (remaining == 1) {
|
||||||
|
utf16EncodedBytesIterator.moveNext();
|
||||||
|
if (replacementCodepoint != null) {
|
||||||
|
_current = replacementCodepoint;
|
||||||
|
return true;
|
||||||
|
} else {
|
||||||
|
throw ArgumentError(
|
||||||
|
'Invalid UTF16 at ${utf16EncodedBytesIterator.position}');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_current = decode();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
int get position => utf16EncodedBytesIterator.position ~/ 2;
|
||||||
|
|
||||||
|
@override
|
||||||
|
void backup([int? by = 1]) {
|
||||||
|
utf16EncodedBytesIterator.backup(2 * by!);
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
int get remaining => (utf16EncodedBytesIterator.remaining + 1) ~/ 2;
|
||||||
|
|
||||||
|
@override
|
||||||
|
void skip([int? count = 1]) {
|
||||||
|
utf16EncodedBytesIterator.skip(2 * count!);
|
||||||
|
}
|
||||||
|
|
||||||
|
int decode();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert UTF-16BE encoded bytes to utf16 code units by grouping 1-2 bytes
|
||||||
|
/// to produce the code unit (0-(2^16)-1).
|
||||||
|
class Utf16beBytesToCodeUnitsDecoder extends Utf16BytesToCodeUnitsDecoder {
|
||||||
|
Utf16beBytesToCodeUnitsDecoder(List<int> utf16EncodedBytes,
|
||||||
|
[int offset = 0,
|
||||||
|
int? length,
|
||||||
|
bool stripBom = true,
|
||||||
|
int replacementCodepoint = unicodeReplacementCharacterCodepoint])
|
||||||
|
: super._fromListRangeIterator(
|
||||||
|
(ListRange(utf16EncodedBytes, offset, length)).iterator,
|
||||||
|
replacementCodepoint) {
|
||||||
|
if (stripBom && hasUtf16BeBom(utf16EncodedBytes, offset, length)) {
|
||||||
|
skip();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
int decode() {
|
||||||
|
utf16EncodedBytesIterator.moveNext();
|
||||||
|
var hi = utf16EncodedBytesIterator.current!;
|
||||||
|
utf16EncodedBytesIterator.moveNext();
|
||||||
|
var lo = utf16EncodedBytesIterator.current!;
|
||||||
|
return (hi << 8) + lo;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert UTF-16LE encoded bytes to utf16 code units by grouping 1-2 bytes
|
||||||
|
/// to produce the code unit (0-(2^16)-1).
|
||||||
|
class Utf16leBytesToCodeUnitsDecoder extends Utf16BytesToCodeUnitsDecoder {
|
||||||
|
Utf16leBytesToCodeUnitsDecoder(List<int> utf16EncodedBytes,
|
||||||
|
[int offset = 0,
|
||||||
|
int? length,
|
||||||
|
bool stripBom = true,
|
||||||
|
int replacementCodepoint = unicodeReplacementCharacterCodepoint])
|
||||||
|
: super._fromListRangeIterator(
|
||||||
|
(ListRange(utf16EncodedBytes, offset, length)).iterator,
|
||||||
|
replacementCodepoint) {
|
||||||
|
if (stripBom && hasUtf16LeBom(utf16EncodedBytes, offset, length)) {
|
||||||
|
skip();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
int decode() {
|
||||||
|
utf16EncodedBytesIterator.moveNext();
|
||||||
|
var lo = utf16EncodedBytesIterator.current!;
|
||||||
|
utf16EncodedBytesIterator.moveNext();
|
||||||
|
var hi = utf16EncodedBytesIterator.current!;
|
||||||
|
return (hi << 8) + lo;
|
||||||
|
}
|
||||||
|
}
|
||||||
319
lib/charset/utf/utf32.dart
Normal file
319
lib/charset/utf/utf32.dart
Normal file
|
|
@ -0,0 +1,319 @@
|
||||||
|
import 'dart:collection';
|
||||||
|
import 'dart:convert';
|
||||||
|
import 'dart:typed_data';
|
||||||
|
|
||||||
|
import 'constants.dart';
|
||||||
|
|
||||||
|
import 'list_range.dart';
|
||||||
|
import 'utils.dart';
|
||||||
|
|
||||||
|
class Utf32Codec extends Encoding {
|
||||||
|
const Utf32Codec();
|
||||||
|
|
||||||
|
@override
|
||||||
|
Converter<List<int>, String> get decoder => const Utf32Decoder();
|
||||||
|
|
||||||
|
@override
|
||||||
|
Converter<String, List<int>> get encoder => const Utf32Encoder();
|
||||||
|
|
||||||
|
@override
|
||||||
|
String get name => 'utf-32';
|
||||||
|
}
|
||||||
|
|
||||||
|
class Utf32Encoder extends Converter<String, List<int>> {
|
||||||
|
const Utf32Encoder();
|
||||||
|
|
||||||
|
/// Produce a list of UTF-32 encoded bytes. This method prefixes the resulting
|
||||||
|
/// bytes with a big-endian byte-order-marker.
|
||||||
|
@override
|
||||||
|
Uint8List convert(String input) => encodeUtf32Be(input, true);
|
||||||
|
|
||||||
|
/// Produce a list of UTF-32BE encoded bytes. By default, this method produces
|
||||||
|
/// UTF-32BE bytes with no BOM.
|
||||||
|
Uint8List encodeUtf32Be(String str, [bool writeBOM = false]) {
|
||||||
|
var utf32CodeUnits = stringToCodepoints(str);
|
||||||
|
var encoding = Uint8List(4 * utf32CodeUnits.length + (writeBOM ? 4 : 0));
|
||||||
|
var i = 0;
|
||||||
|
if (writeBOM) {
|
||||||
|
encoding[i++] = 0;
|
||||||
|
encoding[i++] = 0;
|
||||||
|
encoding[i++] = unicodeUtfBomHi;
|
||||||
|
encoding[i++] = unicodeUtfBomLo;
|
||||||
|
}
|
||||||
|
for (var unit in utf32CodeUnits) {
|
||||||
|
encoding[i++] = (unit! >> 24) & unicodeByteZeroMask;
|
||||||
|
encoding[i++] = (unit >> 16) & unicodeByteZeroMask;
|
||||||
|
encoding[i++] = (unit >> 8) & unicodeByteZeroMask;
|
||||||
|
encoding[i++] = unit & unicodeByteZeroMask;
|
||||||
|
}
|
||||||
|
return encoding;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Produce a list of UTF-32LE encoded bytes. By default, this method produces
|
||||||
|
/// UTF-32BE bytes with no BOM.
|
||||||
|
List<int?> encodeUtf32Le(String str, [bool writeBOM = false]) {
|
||||||
|
var utf32CodeUnits = stringToCodepoints(str);
|
||||||
|
var encoding = <int?>[]..length =
|
||||||
|
4 * utf32CodeUnits.length + (writeBOM ? 4 : 0);
|
||||||
|
var i = 0;
|
||||||
|
if (writeBOM) {
|
||||||
|
encoding[i++] = unicodeUtfBomLo;
|
||||||
|
encoding[i++] = unicodeUtfBomHi;
|
||||||
|
encoding[i++] = 0;
|
||||||
|
encoding[i++] = 0;
|
||||||
|
}
|
||||||
|
for (var unit in utf32CodeUnits) {
|
||||||
|
encoding[i++] = unit! & unicodeByteZeroMask;
|
||||||
|
encoding[i++] = (unit >> 8) & unicodeByteZeroMask;
|
||||||
|
encoding[i++] = (unit >> 16) & unicodeByteZeroMask;
|
||||||
|
encoding[i++] = (unit >> 24) & unicodeByteZeroMask;
|
||||||
|
}
|
||||||
|
return encoding;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class Utf32Decoder extends Converter<List<int>, String> {
|
||||||
|
const Utf32Decoder();
|
||||||
|
|
||||||
|
@override
|
||||||
|
String convert(List<int> input,
|
||||||
|
[int start = 0,
|
||||||
|
int? end,
|
||||||
|
int replacementCodepoint = unicodeReplacementCharacterCodepoint]) {
|
||||||
|
return String.fromCharCodes((Utf32BytesDecoder(input, start,
|
||||||
|
end == null ? input.length : end - start, replacementCodepoint))
|
||||||
|
.decodeRest()
|
||||||
|
.whereType<int>());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Produce a String from a sequence of UTF-32BE encoded bytes. The parameters
|
||||||
|
/// allow an offset into a list of bytes (as int), limiting the length of the
|
||||||
|
/// values be decoded and the ability of override the default Unicode
|
||||||
|
/// replacement character. Set the replacementCharacter to null to throw an
|
||||||
|
/// ArgumentError rather than replace the bad value.
|
||||||
|
String decodeUtf32Be(List<int> bytes,
|
||||||
|
[int offset = 0,
|
||||||
|
int? length,
|
||||||
|
bool stripBom = true,
|
||||||
|
int replacementCodepoint = unicodeReplacementCharacterCodepoint]) =>
|
||||||
|
String.fromCharCodes((Utf32beBytesDecoder(
|
||||||
|
bytes, offset, length, stripBom, replacementCodepoint))
|
||||||
|
.decodeRest()
|
||||||
|
.whereType<int>());
|
||||||
|
|
||||||
|
/// Produce a String from a sequence of UTF-32LE encoded bytes. The parameters
|
||||||
|
/// allow an offset into a list of bytes (as int), limiting the length of the
|
||||||
|
/// values be decoded and the ability of override the default Unicode
|
||||||
|
/// replacement character. Set the replacementCharacter to null to throw an
|
||||||
|
/// ArgumentError rather than replace the bad value.
|
||||||
|
String decodeUtf32Le(List<int> bytes,
|
||||||
|
[int offset = 0,
|
||||||
|
int? length,
|
||||||
|
bool stripBom = true,
|
||||||
|
int replacementCodepoint = unicodeReplacementCharacterCodepoint]) =>
|
||||||
|
String.fromCharCodes((Utf32leBytesDecoder(
|
||||||
|
bytes, offset, length, stripBom, replacementCodepoint))
|
||||||
|
.decodeRest()
|
||||||
|
.whereType<int>());
|
||||||
|
}
|
||||||
|
|
||||||
|
const Utf32Codec utf32 = Utf32Codec();
|
||||||
|
|
||||||
|
/// Identifies whether a List of bytes starts (based on offset) with a
|
||||||
|
/// byte-order marker (BOM).
|
||||||
|
bool hasUtf32Bom(List<int> utf32EncodedBytes, [int offset = 0, int? length]) {
|
||||||
|
return hasUtf32beBom(utf32EncodedBytes, offset, length) ||
|
||||||
|
hasUtf32leBom(utf32EncodedBytes, offset, length);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Identifies whether a List of bytes starts (based on offset) with a
|
||||||
|
/// big-endian byte-order marker (BOM).
|
||||||
|
bool hasUtf32beBom(List<int> utf32EncodedBytes, [int offset = 0, int? length]) {
|
||||||
|
var end = length != null ? offset + length : utf32EncodedBytes.length;
|
||||||
|
return (offset + 4) <= end &&
|
||||||
|
utf32EncodedBytes[offset] == 0 &&
|
||||||
|
utf32EncodedBytes[offset + 1] == 0 &&
|
||||||
|
utf32EncodedBytes[offset + 2] == unicodeUtfBomHi &&
|
||||||
|
utf32EncodedBytes[offset + 3] == unicodeUtfBomLo;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Identifies whether a List of bytes starts (based on offset) with a
|
||||||
|
/// little-endian byte-order marker (BOM).
|
||||||
|
bool hasUtf32leBom(List<int> utf32EncodedBytes, [int offset = 0, int? length]) {
|
||||||
|
var end = length != null ? offset + length : utf32EncodedBytes.length;
|
||||||
|
return (offset + 4) <= end &&
|
||||||
|
utf32EncodedBytes[offset] == unicodeUtfBomLo &&
|
||||||
|
utf32EncodedBytes[offset + 1] == unicodeUtfBomHi &&
|
||||||
|
utf32EncodedBytes[offset + 2] == 0 &&
|
||||||
|
utf32EncodedBytes[offset + 3] == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
typedef Utf32BytesDecoderProvider = Utf32BytesDecoder Function();
|
||||||
|
|
||||||
|
/// Return type of [decodeUtf32AsIterable] and variants. The Iterable type
|
||||||
|
/// provides an iterator on demand and the iterator will only translate bytes
|
||||||
|
/// as requested by the user of the iterator. (Note: results are not cached.)
|
||||||
|
// TODO(floitsch): Consider removing the extend and switch to implements since
|
||||||
|
// that's cheaper to allocate.
|
||||||
|
class IterableUtf32Decoder extends IterableBase<int?> {
|
||||||
|
final Utf32BytesDecoderProvider codeunitsProvider;
|
||||||
|
|
||||||
|
IterableUtf32Decoder._(this.codeunitsProvider);
|
||||||
|
|
||||||
|
@override
|
||||||
|
Utf32BytesDecoder get iterator => codeunitsProvider();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Abstract parent class converts encoded bytes to codepoints.
|
||||||
|
abstract class Utf32BytesDecoder implements ListRangeIterator {
|
||||||
|
// TODO(kevmoo): should this field be private?
|
||||||
|
final ListRangeIterator utf32EncodedBytesIterator;
|
||||||
|
final int? replacementCodepoint;
|
||||||
|
int? _current;
|
||||||
|
|
||||||
|
Utf32BytesDecoder._fromListRangeIterator(
|
||||||
|
this.utf32EncodedBytesIterator, this.replacementCodepoint);
|
||||||
|
|
||||||
|
factory Utf32BytesDecoder(List<int> utf32EncodedBytes,
|
||||||
|
[int offset = 0,
|
||||||
|
int? length,
|
||||||
|
int replacementCodepoint = unicodeReplacementCharacterCodepoint]) {
|
||||||
|
length ??= utf32EncodedBytes.length - offset;
|
||||||
|
if (hasUtf32beBom(utf32EncodedBytes, offset, length)) {
|
||||||
|
return Utf32beBytesDecoder(utf32EncodedBytes, offset + 4, length - 4,
|
||||||
|
false, replacementCodepoint);
|
||||||
|
} else if (hasUtf32leBom(utf32EncodedBytes, offset, length)) {
|
||||||
|
return Utf32leBytesDecoder(utf32EncodedBytes, offset + 4, length - 4,
|
||||||
|
false, replacementCodepoint);
|
||||||
|
} else {
|
||||||
|
return Utf32beBytesDecoder(
|
||||||
|
utf32EncodedBytes, offset, length, false, replacementCodepoint);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
List<int?> decodeRest() {
|
||||||
|
var codeunits = <int?>[]..length = remaining;
|
||||||
|
var i = 0;
|
||||||
|
while (moveNext()) {
|
||||||
|
codeunits[i++] = current;
|
||||||
|
}
|
||||||
|
return codeunits;
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
int? get current => _current;
|
||||||
|
|
||||||
|
@override
|
||||||
|
bool moveNext() {
|
||||||
|
_current = null;
|
||||||
|
var remaining = utf32EncodedBytesIterator.remaining;
|
||||||
|
if (remaining == 0) {
|
||||||
|
_current = null;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (remaining < 4) {
|
||||||
|
utf32EncodedBytesIterator.skip(utf32EncodedBytesIterator.remaining);
|
||||||
|
if (replacementCodepoint != null) {
|
||||||
|
_current = replacementCodepoint;
|
||||||
|
return true;
|
||||||
|
} else {
|
||||||
|
throw ArgumentError(
|
||||||
|
'Invalid UTF32 at ${utf32EncodedBytesIterator.position}');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var codepoint = decode();
|
||||||
|
if (_validCodepoint(codepoint)) {
|
||||||
|
_current = codepoint;
|
||||||
|
return true;
|
||||||
|
} else if (replacementCodepoint != null) {
|
||||||
|
_current = replacementCodepoint;
|
||||||
|
return true;
|
||||||
|
} else {
|
||||||
|
throw ArgumentError(
|
||||||
|
'Invalid UTF32 at ${utf32EncodedBytesIterator.position}');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
int get position => utf32EncodedBytesIterator.position ~/ 4;
|
||||||
|
|
||||||
|
@override
|
||||||
|
void backup([int? by = 1]) {
|
||||||
|
utf32EncodedBytesIterator.backup(4 * by!);
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
int get remaining => (utf32EncodedBytesIterator.remaining + 3) ~/ 4;
|
||||||
|
|
||||||
|
@override
|
||||||
|
void skip([int? count = 1]) {
|
||||||
|
utf32EncodedBytesIterator.skip(4 * count!);
|
||||||
|
}
|
||||||
|
|
||||||
|
int decode();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert UTF-32BE encoded bytes to codepoints by grouping 4 bytes
|
||||||
|
/// to produce the unicode codepoint.
|
||||||
|
class Utf32beBytesDecoder extends Utf32BytesDecoder {
|
||||||
|
Utf32beBytesDecoder(List<int> utf32EncodedBytes,
|
||||||
|
[int offset = 0,
|
||||||
|
int? length,
|
||||||
|
bool stripBom = true,
|
||||||
|
int replacementCodepoint = unicodeReplacementCharacterCodepoint])
|
||||||
|
: super._fromListRangeIterator(
|
||||||
|
(ListRange(utf32EncodedBytes, offset, length)).iterator,
|
||||||
|
replacementCodepoint) {
|
||||||
|
if (stripBom && hasUtf32beBom(utf32EncodedBytes, offset, length)) {
|
||||||
|
skip();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
int decode() {
|
||||||
|
utf32EncodedBytesIterator.moveNext();
|
||||||
|
var value = utf32EncodedBytesIterator.current!;
|
||||||
|
utf32EncodedBytesIterator.moveNext();
|
||||||
|
value = (value << 8) + utf32EncodedBytesIterator.current!;
|
||||||
|
utf32EncodedBytesIterator.moveNext();
|
||||||
|
value = (value << 8) + utf32EncodedBytesIterator.current!;
|
||||||
|
utf32EncodedBytesIterator.moveNext();
|
||||||
|
value = (value << 8) + utf32EncodedBytesIterator.current!;
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert UTF-32BE encoded bytes to codepoints by grouping 4 bytes
|
||||||
|
/// to produce the unicode codepoint.
|
||||||
|
class Utf32leBytesDecoder extends Utf32BytesDecoder {
|
||||||
|
Utf32leBytesDecoder(List<int> utf32EncodedBytes,
|
||||||
|
[int offset = 0,
|
||||||
|
int? length,
|
||||||
|
bool stripBom = true,
|
||||||
|
int replacementCodepoint = unicodeReplacementCharacterCodepoint])
|
||||||
|
: super._fromListRangeIterator(
|
||||||
|
(ListRange(utf32EncodedBytes, offset, length)).iterator,
|
||||||
|
replacementCodepoint) {
|
||||||
|
if (stripBom && hasUtf32leBom(utf32EncodedBytes, offset, length)) {
|
||||||
|
skip();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
int decode() {
|
||||||
|
utf32EncodedBytesIterator.moveNext();
|
||||||
|
var value = utf32EncodedBytesIterator.current!;
|
||||||
|
utf32EncodedBytesIterator.moveNext();
|
||||||
|
value += (utf32EncodedBytesIterator.current! << 8);
|
||||||
|
utf32EncodedBytesIterator.moveNext();
|
||||||
|
value += (utf32EncodedBytesIterator.current! << 16);
|
||||||
|
utf32EncodedBytesIterator.moveNext();
|
||||||
|
value += (utf32EncodedBytesIterator.current! << 24);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool _validCodepoint(int codepoint) {
|
||||||
|
return (codepoint >= 0 && codepoint < unicodeUtf16ReservedLo) ||
|
||||||
|
(codepoint > unicodeUtf16ReservedHi && codepoint < unicodeValidRangeMax);
|
||||||
|
}
|
||||||
75
lib/charset/utf/utf_16_code_unit_decoder.dart
Normal file
75
lib/charset/utf/utf_16_code_unit_decoder.dart
Normal file
|
|
@ -0,0 +1,75 @@
|
||||||
|
import 'constants.dart';
|
||||||
|
import 'list_range.dart';
|
||||||
|
|
||||||
|
/// An Iterator<int> of codepoints built on an Iterator of UTF-16 code units.
|
||||||
|
/// The parameters can override the default Unicode replacement character. Set
|
||||||
|
/// the replacementCharacter to null to throw an ArgumentError
|
||||||
|
/// rather than replace the bad value.
|
||||||
|
class Utf16CodeUnitDecoder implements Iterator<int?> {
|
||||||
|
// TODO(kevmoo): should this field be private?
|
||||||
|
final ListRangeIterator utf16CodeUnitIterator;
|
||||||
|
final int? replacementCodepoint;
|
||||||
|
int? _current;
|
||||||
|
|
||||||
|
Utf16CodeUnitDecoder(List<int> utf16CodeUnits,
|
||||||
|
[int offset = 0,
|
||||||
|
int? length,
|
||||||
|
this.replacementCodepoint = unicodeReplacementCharacterCodepoint])
|
||||||
|
: utf16CodeUnitIterator =
|
||||||
|
(ListRange(utf16CodeUnits, offset, length)).iterator;
|
||||||
|
|
||||||
|
Utf16CodeUnitDecoder.fromListRangeIterator(
|
||||||
|
this.utf16CodeUnitIterator, this.replacementCodepoint);
|
||||||
|
|
||||||
|
Iterator<int?> get iterator => this;
|
||||||
|
|
||||||
|
@override
|
||||||
|
int? get current => _current;
|
||||||
|
|
||||||
|
@override
|
||||||
|
bool moveNext() {
|
||||||
|
_current = null;
|
||||||
|
if (!utf16CodeUnitIterator.moveNext()) return false;
|
||||||
|
|
||||||
|
var value = utf16CodeUnitIterator.current!;
|
||||||
|
if (value < 0) {
|
||||||
|
if (replacementCodepoint != null) {
|
||||||
|
_current = replacementCodepoint;
|
||||||
|
} else {
|
||||||
|
throw ArgumentError(
|
||||||
|
'Invalid UTF16 at ${utf16CodeUnitIterator.position}');
|
||||||
|
}
|
||||||
|
} else if (value < unicodeUtf16ReservedLo ||
|
||||||
|
(value > unicodeUtf16ReservedHi && value <= unicodePlaneOneMax)) {
|
||||||
|
// transfer directly
|
||||||
|
_current = value;
|
||||||
|
} else if (value < unicodeUtf16SurrogateUnit1Base &&
|
||||||
|
utf16CodeUnitIterator.moveNext()) {
|
||||||
|
// merge surrogate pair
|
||||||
|
var nextValue = utf16CodeUnitIterator.current!;
|
||||||
|
if (nextValue >= unicodeUtf16SurrogateUnit1Base &&
|
||||||
|
nextValue <= unicodeUtf16ReservedHi) {
|
||||||
|
value = (value - unicodeUtf16SurrogateUnit0Base) << 10;
|
||||||
|
value +=
|
||||||
|
unicodeUtf16Offset + (nextValue - unicodeUtf16SurrogateUnit1Base);
|
||||||
|
_current = value;
|
||||||
|
} else {
|
||||||
|
if (nextValue >= unicodeUtf16SurrogateUnit0Base &&
|
||||||
|
nextValue < unicodeUtf16SurrogateUnit1Base) {
|
||||||
|
utf16CodeUnitIterator.backup();
|
||||||
|
}
|
||||||
|
if (replacementCodepoint != null) {
|
||||||
|
_current = replacementCodepoint;
|
||||||
|
} else {
|
||||||
|
throw ArgumentError(
|
||||||
|
'Invalid UTF16 at ${utf16CodeUnitIterator.position}');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (replacementCodepoint != null) {
|
||||||
|
_current = replacementCodepoint;
|
||||||
|
} else {
|
||||||
|
throw ArgumentError('Invalid UTF16 at ${utf16CodeUnitIterator.position}');
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
79
lib/charset/utf/utils.dart
Normal file
79
lib/charset/utf/utils.dart
Normal file
|
|
@ -0,0 +1,79 @@
|
||||||
|
import 'constants.dart';
|
||||||
|
import 'list_range.dart';
|
||||||
|
import 'utf_16_code_unit_decoder.dart';
|
||||||
|
|
||||||
|
/// TODO(jmesserly): would be nice to have this on String (dartbug.com/6501).
|
||||||
|
/// Provide a list of Unicode codepoints for a given string.
|
||||||
|
List<int?> stringToCodepoints(String str) {
|
||||||
|
// Note: str.codeUnits gives us 16-bit code units on all Dart implementations.
|
||||||
|
// So we need to convert.
|
||||||
|
return utf16CodeUnitsToCodepoints(str.codeUnits);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decodes the utf16 codeunits to codepoints.
|
||||||
|
List<int?> utf16CodeUnitsToCodepoints(List<int?> utf16CodeUnits,
|
||||||
|
[int offset = 0,
|
||||||
|
int? length,
|
||||||
|
int replacementCodepoint = unicodeReplacementCharacterCodepoint]) {
|
||||||
|
var source = (ListRange(utf16CodeUnits, offset, length)).iterator;
|
||||||
|
var decoder =
|
||||||
|
Utf16CodeUnitDecoder.fromListRangeIterator(source, replacementCodepoint);
|
||||||
|
var codepoints = <int?>[]..length = source.remaining;
|
||||||
|
var i = 0;
|
||||||
|
while (decoder.moveNext()) {
|
||||||
|
codepoints[i++] = decoder.current;
|
||||||
|
}
|
||||||
|
if (i == codepoints.length) {
|
||||||
|
return codepoints;
|
||||||
|
} else {
|
||||||
|
var codepointTrunc = <int?>[]..length = i;
|
||||||
|
codepointTrunc.setRange(0, i, codepoints);
|
||||||
|
return codepointTrunc;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Encode code points as UTF16 code units.
|
||||||
|
List<int?> codepointsToUtf16CodeUnits(List<int> codepoints,
|
||||||
|
[int offset = 0,
|
||||||
|
int? length,
|
||||||
|
int? replacementCodepoint = unicodeReplacementCharacterCodepoint]) {
|
||||||
|
var listRange = ListRange(codepoints, offset, length);
|
||||||
|
var encodedLength = 0;
|
||||||
|
for (var value in listRange) {
|
||||||
|
if (null == value) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((value >= 0 && value < unicodeUtf16ReservedLo) ||
|
||||||
|
(value > unicodeUtf16ReservedHi && value <= unicodePlaneOneMax)) {
|
||||||
|
encodedLength++;
|
||||||
|
} else if (value > unicodePlaneOneMax && value <= unicodeValidRangeMax) {
|
||||||
|
encodedLength += 2;
|
||||||
|
} else {
|
||||||
|
encodedLength++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var codeUnitsBuffer = <int?>[]..length = encodedLength;
|
||||||
|
var j = 0;
|
||||||
|
for (var value in listRange) {
|
||||||
|
if (null == value) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if ((value >= 0 && value < unicodeUtf16ReservedLo) ||
|
||||||
|
(value > unicodeUtf16ReservedHi && value <= unicodePlaneOneMax)) {
|
||||||
|
codeUnitsBuffer[j++] = value;
|
||||||
|
} else if (value > unicodePlaneOneMax && value <= unicodeValidRangeMax) {
|
||||||
|
var base = value - unicodeUtf16Offset;
|
||||||
|
codeUnitsBuffer[j++] =
|
||||||
|
unicodeUtf16SurrogateUnit0Base + ((base & unicodeUtf16HiMask) >> 10);
|
||||||
|
codeUnitsBuffer[j++] =
|
||||||
|
unicodeUtf16SurrogateUnit1Base + (base & unicodeUtf16LoMask);
|
||||||
|
} else if (replacementCodepoint != null) {
|
||||||
|
codeUnitsBuffer[j++] = replacementCodepoint;
|
||||||
|
} else {
|
||||||
|
throw ArgumentError('Invalid encoding');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return codeUnitsBuffer;
|
||||||
|
}
|
||||||
34652
lib/code_map.dart
Normal file
34652
lib/code_map.dart
Normal file
File diff suppressed because it is too large
Load diff
59
lib/cp949.dart
Normal file
59
lib/cp949.dart
Normal file
|
|
@ -0,0 +1,59 @@
|
||||||
|
import 'dart:typed_data';
|
||||||
|
|
||||||
|
import 'code_map.dart' show cp949ToUnicodeCodeMap, unicodeToCp949CodeMap;
|
||||||
|
|
||||||
|
Uint8List toUnicodes(final Uint8List codeUnits) {
|
||||||
|
final unicodes = Uint8List(codeUnits.length);
|
||||||
|
var j = 0;
|
||||||
|
for (var i = 0; i < codeUnits.length; j++) {
|
||||||
|
// When 1 byte becomes 1 cp949 code
|
||||||
|
if (0x00 <= codeUnits[i] && codeUnits[i] <= 0x7F) {
|
||||||
|
unicodes[j] = cp949ToUnicodeCodeMap[codeUnits[i]]!;
|
||||||
|
i = i + 1;
|
||||||
|
} else {
|
||||||
|
// When 2 bytes become 1 cp949 code
|
||||||
|
final cp949Code = (codeUnits[i] << 8) + codeUnits[i + 1];
|
||||||
|
i = i + 2;
|
||||||
|
if (0x8141 <= cp949Code &&
|
||||||
|
cp949Code <= 0xFDFE &&
|
||||||
|
cp949ToUnicodeCodeMap.containsKey(cp949Code)) {
|
||||||
|
unicodes[j] = cp949ToUnicodeCodeMap[cp949Code]!;
|
||||||
|
} else {
|
||||||
|
throw FormatException(
|
||||||
|
'Invalid code unit of CP949. It has to be (>=0x00 <=0x7F) || (>=0x8141 <=0xFDFE).');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return unicodes.sublist(0, j);
|
||||||
|
}
|
||||||
|
|
||||||
|
String decode(final Uint8List codeUnits) {
|
||||||
|
return String.fromCharCodes(toUnicodes(codeUnits));
|
||||||
|
}
|
||||||
|
|
||||||
|
String decodeString(final String brokenString) {
|
||||||
|
return decode(Uint8List.fromList(brokenString.codeUnits));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Dart does not support non-unicode encoding.
|
||||||
|
/// Thus, the return value has to be raw byte array of CP949.
|
||||||
|
Uint8List encode(final String str) {
|
||||||
|
final cp949codeUnits = Uint8List(str.codeUnits.length * 2);
|
||||||
|
var i = 0;
|
||||||
|
for (final unicode in str.codeUnits) {
|
||||||
|
final cp949Code = unicodeToCp949CodeMap[unicode];
|
||||||
|
final firstByte = cp949Code! >> 8;
|
||||||
|
final secondByte = cp949Code - (firstByte << 8);
|
||||||
|
if (firstByte != 0) {
|
||||||
|
cp949codeUnits[i] = firstByte;
|
||||||
|
i = i + 1;
|
||||||
|
}
|
||||||
|
cp949codeUnits[i] = secondByte;
|
||||||
|
i = i + 1;
|
||||||
|
}
|
||||||
|
return cp949codeUnits.sublist(0, i);
|
||||||
|
}
|
||||||
|
|
||||||
|
String encodeToString(final String str) {
|
||||||
|
return String.fromCharCodes(encode(str));
|
||||||
|
}
|
||||||
134
lib/subtitle/ParserSRT.dart
Normal file
134
lib/subtitle/ParserSRT.dart
Normal file
|
|
@ -0,0 +1,134 @@
|
||||||
|
// ignore: file_names
|
||||||
|
class ParserSRT {
|
||||||
|
Subtitles getSubtitlesData(
|
||||||
|
String subtitlesContent,
|
||||||
|
SubtitleType subtitleType,
|
||||||
|
) {
|
||||||
|
RegExp regExp;
|
||||||
|
if (subtitleType == SubtitleType.webvtt) {
|
||||||
|
regExp = RegExp(
|
||||||
|
r'((\d{2}):(\d{2}):(\d{2})\.(\d+)) +--> +((\d{2}):(\d{2}):(\d{2})\.(\d{3})).*[\r\n]+\s*((?:(?!\r?\n\r?).)*(\r\n|\r|\n)(?:.*))',
|
||||||
|
caseSensitive: false,
|
||||||
|
multiLine: true,
|
||||||
|
);
|
||||||
|
} else if (subtitleType == SubtitleType.srt) {
|
||||||
|
regExp = RegExp(
|
||||||
|
r'((\d{2}):(\d{2}):(\d{2})\,(\d+)) +--> +((\d{2}):(\d{2}):(\d{2})\,(\d{3})).*[\r\n]+\s*((?:(?!\r?\n\r?).)*(\r\n|\r|\n)(?:.*))',
|
||||||
|
caseSensitive: false,
|
||||||
|
multiLine: true,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
throw Exception('Incorrect subtitle type');
|
||||||
|
}
|
||||||
|
|
||||||
|
final matches = regExp.allMatches(subtitlesContent).toList();
|
||||||
|
final subtitleList = <Subtitle>[];
|
||||||
|
|
||||||
|
for (final regExpMatch in matches) {
|
||||||
|
final startTimeHours = int.parse(regExpMatch.group(2)!);
|
||||||
|
final startTimeMinutes = int.parse(regExpMatch.group(3)!);
|
||||||
|
final startTimeSeconds = int.parse(regExpMatch.group(4)!);
|
||||||
|
final startTimeMilliseconds = int.parse(regExpMatch.group(5)!);
|
||||||
|
|
||||||
|
final endTimeHours = int.parse(regExpMatch.group(7)!);
|
||||||
|
final endTimeMinutes = int.parse(regExpMatch.group(8)!);
|
||||||
|
final endTimeSeconds = int.parse(regExpMatch.group(9)!);
|
||||||
|
final endTimeMilliseconds = int.parse(regExpMatch.group(10)!);
|
||||||
|
final text = removeAllHtmlTags(regExpMatch.group(11)!);
|
||||||
|
|
||||||
|
final startTime = Time(Duration(
|
||||||
|
hours: startTimeHours,
|
||||||
|
minutes: startTimeMinutes,
|
||||||
|
seconds: startTimeSeconds,
|
||||||
|
milliseconds: startTimeMilliseconds,
|
||||||
|
));
|
||||||
|
final endTime = Time(Duration(
|
||||||
|
hours: endTimeHours,
|
||||||
|
minutes: endTimeMinutes,
|
||||||
|
seconds: endTimeSeconds,
|
||||||
|
milliseconds: endTimeMilliseconds,
|
||||||
|
));
|
||||||
|
|
||||||
|
subtitleList.add(
|
||||||
|
Subtitle(startTime: startTime, endTime: endTime, text: text.trim()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return Subtitles(subtitles: subtitleList);
|
||||||
|
}
|
||||||
|
|
||||||
|
String removeAllHtmlTags(String htmlText) {
|
||||||
|
final exp = RegExp(
|
||||||
|
'(<[^>]*>)',
|
||||||
|
multiLine: true,
|
||||||
|
);
|
||||||
|
var newHtmlText = htmlText;
|
||||||
|
exp.allMatches(htmlText).toList().forEach(
|
||||||
|
(RegExpMatch regExpMatch) {
|
||||||
|
newHtmlText = regExpMatch.group(0) == '<br>'
|
||||||
|
? newHtmlText.replaceAll(regExpMatch.group(0)!, '\n')
|
||||||
|
: newHtmlText.replaceAll(regExpMatch.group(0)!, '');
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
return newHtmlText;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class Subtitles {
|
||||||
|
Subtitles({
|
||||||
|
required this.subtitles,
|
||||||
|
});
|
||||||
|
final List<Subtitle> subtitles;
|
||||||
|
}
|
||||||
|
|
||||||
|
class Subtitle {
|
||||||
|
Subtitle({
|
||||||
|
required this.startTime,
|
||||||
|
required this.endTime,
|
||||||
|
required this.text,
|
||||||
|
});
|
||||||
|
Time startTime;
|
||||||
|
Time endTime;
|
||||||
|
String text;
|
||||||
|
|
||||||
|
List<Object?> get props => [startTime, endTime, text];
|
||||||
|
}
|
||||||
|
|
||||||
|
enum SubtitleType {
|
||||||
|
webvtt,
|
||||||
|
srt,
|
||||||
|
}
|
||||||
|
|
||||||
|
class Time {
|
||||||
|
Time(Duration duration) {
|
||||||
|
this.duration = duration;
|
||||||
|
}
|
||||||
|
|
||||||
|
late Duration _duration;
|
||||||
|
Duration get duration => _duration;
|
||||||
|
set duration(Duration value) {
|
||||||
|
_duration = value;
|
||||||
|
var timeStr = duration.toString();
|
||||||
|
var timeArr = timeStr.substring(0, timeStr.lastIndexOf('.') + 4).split(':');
|
||||||
|
hour = int.parse(timeArr[0]);
|
||||||
|
minutes = int.parse(timeArr[1]);
|
||||||
|
var secArr = timeArr[2].split('.');
|
||||||
|
seconds = int.parse(secArr[0]);
|
||||||
|
milliseconds = int.parse(secArr[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
late int hour;
|
||||||
|
late int minutes;
|
||||||
|
late int seconds;
|
||||||
|
late int milliseconds;
|
||||||
|
|
||||||
|
Duration get before {
|
||||||
|
return Duration(milliseconds: duration.inMilliseconds - 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
String toString() {
|
||||||
|
return '${hour.toString().padLeft(2, '0')}:${minutes.toString().padLeft(2, '0')}:${seconds.toString().padLeft(2, '0')}.${milliseconds.toString().padLeft(3, '0')}';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -16,6 +16,8 @@ dependencies:
|
||||||
yaml: ^3.1.1
|
yaml: ^3.1.1
|
||||||
http: ^0.13.5
|
http: ^0.13.5
|
||||||
color: ^3.0.0
|
color: ^3.0.0
|
||||||
|
# subtitle_wrapper_package: ^2.1.1
|
||||||
|
# equatable: ^2.0.5
|
||||||
|
|
||||||
dev_dependencies:
|
dev_dependencies:
|
||||||
lints: ^2.0.0
|
lints: ^2.0.0
|
||||||
|
|
|
||||||
3
test_eucjp.smi
Normal file
3
test_eucjp.smi
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
OS だと iOS SDK, Android だと Java 組み込みの機能をそのまま使って変換しているので幅広い文字コードに対応しており、バグも恐らく少なくそれでいて高速であるため。
|
||||||
|
逆に、これ以外のプラットフォームでは現状動作しない（macOS とか Windows）ので、現時点であまりいないとは思われますが、デスクトップアプリもサポートしたい場合には使えません。
|
||||||
|
そういう人向けには、代わりに以下が使えます。
|
||||||
4844
test_euckr.smi
Normal file
4844
test_euckr.smi
Normal file
File diff suppressed because it is too large
Load diff
BIN
test_utf16le.ass
Normal file
BIN
test_utf16le.ass
Normal file
Binary file not shown.
BIN
test_utf16le.smi
Normal file
BIN
test_utf16le.smi
Normal file
Binary file not shown.
66
test_utf8bom.srt
Normal file
66
test_utf8bom.srt
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
1
|
||||||
|
00:01:00,640 --> 00:01:02,529
|
||||||
|
이것은 테스트 입니다
|
||||||
|
|
||||||
|
2
|
||||||
|
00:01:02,600 --> 00:01:04,648
|
||||||
|
(CROWD CHEERING)
|
||||||
|
|
||||||
|
3
|
||||||
|
00:02:17,080 --> 00:02:19,128
|
||||||
|
(MUSIC PLAYING)
|
||||||
|
|
||||||
|
4
|
||||||
|
00:02:52,320 --> 00:02:54,129
|
||||||
|
(WHISPERING)
|
||||||
|
|
||||||
|
5
|
||||||
|
00:03:00,560 --> 00:03:01,686
|
||||||
|
(ELEVATOR BELL DINGS)
|
||||||
|
|
||||||
|
6
|
||||||
|
00:03:51,480 --> 00:03:52,766
|
||||||
|
Where are you going?
|
||||||
|
|
||||||
|
7
|
||||||
|
00:03:55,200 --> 00:03:56,486
|
||||||
|
I won't be long.
|
||||||
|
|
||||||
|
8
|
||||||
|
00:04:59,720 --> 00:05:00,801
|
||||||
|
(MAN SPEAKING ITALIAN)
|
||||||
|
|
||||||
|
9
|
||||||
|
00:05:00,880 --> 00:05:02,484
|
||||||
|
Welcome, Signor Sciarra.
|
||||||
|
|
||||||
|
10
|
||||||
|
00:05:02,920 --> 00:05:05,048
|
||||||
|
I trust you had a pleasantjourney.
|
||||||
|
|
||||||
|
11
|
||||||
|
00:05:10,440 --> 00:05:11,487
|
||||||
|
(MARCO SPEAKING)
|
||||||
|
|
||||||
|
12
|
||||||
|
02:20:28,000 --> 02:20:30,002
|
||||||
|
What are you doing here?
|
||||||
|
Good morning, Q.
|
||||||
|
|
||||||
|
13
|
||||||
|
02:20:37,880 --> 02:20:39,086
|
||||||
|
I thought you'd gone.
|
||||||
|
|
||||||
|
14
|
||||||
|
02:20:39,200 --> 02:20:40,326
|
||||||
|
I have.
|
||||||
|
|
||||||
|
15
|
||||||
|
02:20:42,200 --> 02:20:44,441
|
||||||
|
There's just
|
||||||
|
one thing I need.
|
||||||
|
|
||||||
|
16
|
||||||
|
02:20:48,880 --> 02:20:51,451
|
||||||
|
(ENGINE REVVING)
|
||||||
|
|
||||||
Loading…
Reference in a new issue