utf8.js 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227
  1. "use strict";
  2. Object.defineProperty(exports, "__esModule", { value: true });
  3. exports.toUtf8CodePoints = exports.toUtf8String = exports.toUtf8Bytes = exports.Utf8ErrorFuncs = void 0;
  4. /**
  5. * Using strings in Ethereum (or any security-basd system) requires
  6. * additional care. These utilities attempt to mitigate some of the
  7. * safety issues as well as provide the ability to recover and analyse
  8. * strings.
  9. *
  10. * @_subsection api/utils:Strings and UTF-8 [about-strings]
  11. */
  12. const data_js_1 = require("./data.js");
  13. const errors_js_1 = require("./errors.js");
  14. function errorFunc(reason, offset, bytes, output, badCodepoint) {
  15. (0, errors_js_1.assertArgument)(false, `invalid codepoint at offset ${offset}; ${reason}`, "bytes", bytes);
  16. }
  17. function ignoreFunc(reason, offset, bytes, output, badCodepoint) {
  18. // If there is an invalid prefix (including stray continuation), skip any additional continuation bytes
  19. if (reason === "BAD_PREFIX" || reason === "UNEXPECTED_CONTINUE") {
  20. let i = 0;
  21. for (let o = offset + 1; o < bytes.length; o++) {
  22. if (bytes[o] >> 6 !== 0x02) {
  23. break;
  24. }
  25. i++;
  26. }
  27. return i;
  28. }
  29. // This byte runs us past the end of the string, so just jump to the end
  30. // (but the first byte was read already read and therefore skipped)
  31. if (reason === "OVERRUN") {
  32. return bytes.length - offset - 1;
  33. }
  34. // Nothing to skip
  35. return 0;
  36. }
  37. function replaceFunc(reason, offset, bytes, output, badCodepoint) {
  38. // Overlong representations are otherwise "valid" code points; just non-deistingtished
  39. if (reason === "OVERLONG") {
  40. (0, errors_js_1.assertArgument)(typeof (badCodepoint) === "number", "invalid bad code point for replacement", "badCodepoint", badCodepoint);
  41. output.push(badCodepoint);
  42. return 0;
  43. }
  44. // Put the replacement character into the output
  45. output.push(0xfffd);
  46. // Otherwise, process as if ignoring errors
  47. return ignoreFunc(reason, offset, bytes, output, badCodepoint);
  48. }
  49. /**
  50. * A handful of popular, built-in UTF-8 error handling strategies.
  51. *
  52. * **``"error"``** - throws on ANY illegal UTF-8 sequence or
  53. * non-canonical (overlong) codepoints (this is the default)
  54. *
  55. * **``"ignore"``** - silently drops any illegal UTF-8 sequence
  56. * and accepts non-canonical (overlong) codepoints
  57. *
  58. * **``"replace"``** - replace any illegal UTF-8 sequence with the
  59. * UTF-8 replacement character (i.e. ``"\\ufffd"``) and accepts
  60. * non-canonical (overlong) codepoints
  61. *
  62. * @returns: Record<"error" | "ignore" | "replace", Utf8ErrorFunc>
  63. */
  64. exports.Utf8ErrorFuncs = Object.freeze({
  65. error: errorFunc,
  66. ignore: ignoreFunc,
  67. replace: replaceFunc
  68. });
  69. // http://stackoverflow.com/questions/13356493/decode-utf-8-with-javascript#13691499
  70. function getUtf8CodePoints(_bytes, onError) {
  71. if (onError == null) {
  72. onError = exports.Utf8ErrorFuncs.error;
  73. }
  74. const bytes = (0, data_js_1.getBytes)(_bytes, "bytes");
  75. const result = [];
  76. let i = 0;
  77. // Invalid bytes are ignored
  78. while (i < bytes.length) {
  79. const c = bytes[i++];
  80. // 0xxx xxxx
  81. if (c >> 7 === 0) {
  82. result.push(c);
  83. continue;
  84. }
  85. // Multibyte; how many bytes left for this character?
  86. let extraLength = null;
  87. let overlongMask = null;
  88. // 110x xxxx 10xx xxxx
  89. if ((c & 0xe0) === 0xc0) {
  90. extraLength = 1;
  91. overlongMask = 0x7f;
  92. // 1110 xxxx 10xx xxxx 10xx xxxx
  93. }
  94. else if ((c & 0xf0) === 0xe0) {
  95. extraLength = 2;
  96. overlongMask = 0x7ff;
  97. // 1111 0xxx 10xx xxxx 10xx xxxx 10xx xxxx
  98. }
  99. else if ((c & 0xf8) === 0xf0) {
  100. extraLength = 3;
  101. overlongMask = 0xffff;
  102. }
  103. else {
  104. if ((c & 0xc0) === 0x80) {
  105. i += onError("UNEXPECTED_CONTINUE", i - 1, bytes, result);
  106. }
  107. else {
  108. i += onError("BAD_PREFIX", i - 1, bytes, result);
  109. }
  110. continue;
  111. }
  112. // Do we have enough bytes in our data?
  113. if (i - 1 + extraLength >= bytes.length) {
  114. i += onError("OVERRUN", i - 1, bytes, result);
  115. continue;
  116. }
  117. // Remove the length prefix from the char
  118. let res = c & ((1 << (8 - extraLength - 1)) - 1);
  119. for (let j = 0; j < extraLength; j++) {
  120. let nextChar = bytes[i];
  121. // Invalid continuation byte
  122. if ((nextChar & 0xc0) != 0x80) {
  123. i += onError("MISSING_CONTINUE", i, bytes, result);
  124. res = null;
  125. break;
  126. }
  127. ;
  128. res = (res << 6) | (nextChar & 0x3f);
  129. i++;
  130. }
  131. // See above loop for invalid continuation byte
  132. if (res === null) {
  133. continue;
  134. }
  135. // Maximum code point
  136. if (res > 0x10ffff) {
  137. i += onError("OUT_OF_RANGE", i - 1 - extraLength, bytes, result, res);
  138. continue;
  139. }
  140. // Reserved for UTF-16 surrogate halves
  141. if (res >= 0xd800 && res <= 0xdfff) {
  142. i += onError("UTF16_SURROGATE", i - 1 - extraLength, bytes, result, res);
  143. continue;
  144. }
  145. // Check for overlong sequences (more bytes than needed)
  146. if (res <= overlongMask) {
  147. i += onError("OVERLONG", i - 1 - extraLength, bytes, result, res);
  148. continue;
  149. }
  150. result.push(res);
  151. }
  152. return result;
  153. }
  154. // http://stackoverflow.com/questions/18729405/how-to-convert-utf8-string-to-byte-array
  155. /**
  156. * Returns the UTF-8 byte representation of %%str%%.
  157. *
  158. * If %%form%% is specified, the string is normalized.
  159. */
  160. function toUtf8Bytes(str, form) {
  161. (0, errors_js_1.assertArgument)(typeof (str) === "string", "invalid string value", "str", str);
  162. if (form != null) {
  163. (0, errors_js_1.assertNormalize)(form);
  164. str = str.normalize(form);
  165. }
  166. let result = [];
  167. for (let i = 0; i < str.length; i++) {
  168. const c = str.charCodeAt(i);
  169. if (c < 0x80) {
  170. result.push(c);
  171. }
  172. else if (c < 0x800) {
  173. result.push((c >> 6) | 0xc0);
  174. result.push((c & 0x3f) | 0x80);
  175. }
  176. else if ((c & 0xfc00) == 0xd800) {
  177. i++;
  178. const c2 = str.charCodeAt(i);
  179. (0, errors_js_1.assertArgument)(i < str.length && ((c2 & 0xfc00) === 0xdc00), "invalid surrogate pair", "str", str);
  180. // Surrogate Pair
  181. const pair = 0x10000 + ((c & 0x03ff) << 10) + (c2 & 0x03ff);
  182. result.push((pair >> 18) | 0xf0);
  183. result.push(((pair >> 12) & 0x3f) | 0x80);
  184. result.push(((pair >> 6) & 0x3f) | 0x80);
  185. result.push((pair & 0x3f) | 0x80);
  186. }
  187. else {
  188. result.push((c >> 12) | 0xe0);
  189. result.push(((c >> 6) & 0x3f) | 0x80);
  190. result.push((c & 0x3f) | 0x80);
  191. }
  192. }
  193. return new Uint8Array(result);
  194. }
  195. exports.toUtf8Bytes = toUtf8Bytes;
  196. ;
  197. //export
  198. function _toUtf8String(codePoints) {
  199. return codePoints.map((codePoint) => {
  200. if (codePoint <= 0xffff) {
  201. return String.fromCharCode(codePoint);
  202. }
  203. codePoint -= 0x10000;
  204. return String.fromCharCode((((codePoint >> 10) & 0x3ff) + 0xd800), ((codePoint & 0x3ff) + 0xdc00));
  205. }).join("");
  206. }
  207. /**
  208. * Returns the string represented by the UTF-8 data %%bytes%%.
  209. *
  210. * When %%onError%% function is specified, it is called on UTF-8
  211. * errors allowing recovery using the [[Utf8ErrorFunc]] API.
  212. * (default: [error](Utf8ErrorFuncs))
  213. */
  214. function toUtf8String(bytes, onError) {
  215. return _toUtf8String(getUtf8CodePoints(bytes, onError));
  216. }
  217. exports.toUtf8String = toUtf8String;
  218. /**
  219. * Returns the UTF-8 code-points for %%str%%.
  220. *
  221. * If %%form%% is specified, the string is normalized.
  222. */
  223. function toUtf8CodePoints(str, form) {
  224. return getUtf8CodePoints(toUtf8Bytes(str, form));
  225. }
  226. exports.toUtf8CodePoints = toUtf8CodePoints;
  227. //# sourceMappingURL=utf8.js.map