Source: lib/util/string_utils.js

  1. /*! @license
  2.  * Shaka Player
  3.  * Copyright 2016 Google LLC
  4.  * SPDX-License-Identifier: Apache-2.0
  5.  */
  7. goog.provide('shaka.util.StringUtils');
  9. goog.require('goog.asserts');
  10. goog.require('shaka.device.DeviceFactory');
  11. goog.require('shaka.log');
  12. goog.require('shaka.util.BufferUtils');
  13. goog.require('shaka.util.Error');
  14. goog.require('shaka.util.Lazy');
  17. /**
  18.  * @namespace shaka.util.StringUtils
  19.  * @summary A set of string utility functions.
  20.  * @export
  21.  */
  22. shaka.util.StringUtils = class {
  23.   /**
  24.    * Creates a string from the given buffer as UTF-8 encoding.
  25.    *
  26.    * @param {?BufferSource} data
  27.    * @return {string}
  28.    * @export
  29.    */
  30.   static fromUTF8(data) {
  31.     if (!data) {
  32.       return '';
  33.     }
  35.     let uint8 = shaka.util.BufferUtils.toUint8(data);
  36.     // If present, strip off the UTF-8 BOM.
  37.     if (uint8[0] == 0xef && uint8[1] == 0xbb && uint8[2] == 0xbf) {
  38.       uint8 = uint8.subarray(3);
  39.     }
  41.     if (window.TextDecoder && !shaka.device.DeviceFactory.getDevice()
  42.         .shouldAvoidUseTextDecoderEncoder()) {
  43.       // Use the TextDecoder interface to decode the text.  This has the
  44.       // advantage compared to the previously-standard decodeUriComponent that
  45.       // it will continue parsing even if it finds an invalid UTF8 character,
  46.       // rather than stop and throw an error.
  47.       const utf8decoder = new TextDecoder();
  48.       const decoded = utf8decoder.decode(uint8);
  49.       if (decoded.includes('\uFFFD')) {
  50.         shaka.log.alwaysError('Decoded string contains an "unknown character' +
  51.                               '" codepoint.  That probably means the UTF8 ' +
  52.                               'encoding was incorrect!');
  53.       }
  54.       return decoded;
  55.     } else {
  56.       // Homebrewed UTF-8 decoder based on
  57.       // https://en.wikipedia.org/wiki/UTF-8#Encoding
  58.       // Unlike decodeURIComponent, won't throw on bad encoding.
  59.       // In this way, it is similar to TextDecoder.
  61.       let decoded = '';
  62.       for (let i = 0; i < uint8.length; ++i) {
  63.         // By default, the "replacement character" codepoint.
  64.         let codePoint = 0xFFFD;
  66.         // Top bit is 0, 1-byte encoding.
  67.         if ((uint8[i] & 0x80) == 0) {
  68.           codePoint = uint8[i];
  70.         // Top 3 bits of byte 0 are 110, top 2 bits of byte 1 are 10,
  71.         // 2-byte encoding.
  72.         } else if (uint8.length >= i + 2 &&
  73.             (uint8[i] & 0xe0) == 0xc0 &&
  74.             (uint8[i + 1] & 0xc0) == 0x80) {
  75.           codePoint = ((uint8[i] & 0x1f) << 6) |
  76.                       ((uint8[i + 1] & 0x3f));
  77.           i += 1;  // Consume one extra byte.
  79.         // Top 4 bits of byte 0 are 1110, top 2 bits of byte 1 and 2 are 10,
  80.         // 3-byte encoding.
  81.         } else if (uint8.length >= i + 3 &&
  82.             (uint8[i] & 0xf0) == 0xe0 &&
  83.             (uint8[i + 1] & 0xc0) == 0x80 &&
  84.             (uint8[i + 2] & 0xc0) == 0x80) {
  85.           codePoint = ((uint8[i] & 0x0f) << 12) |
  86.                       ((uint8[i + 1] & 0x3f) << 6) |
  87.                       ((uint8[i + 2] & 0x3f));
  88.           i += 2;  // Consume two extra bytes.
  90.         // Top 5 bits of byte 0 are 11110, top 2 bits of byte 1, 2 and 3 are 10,
  91.         // 4-byte encoding.
  92.         } else if (uint8.length >= i + 4 &&
  93.             (uint8[i] & 0xf1) == 0xf0 &&
  94.             (uint8[i + 1] & 0xc0) == 0x80 &&
  95.             (uint8[i + 2] & 0xc0) == 0x80 &&
  96.             (uint8[i + 3] & 0xc0) == 0x80) {
  97.           codePoint = ((uint8[i] & 0x07) << 18) |
  98.                       ((uint8[i + 1] & 0x3f) << 12) |
  99.                       ((uint8[i + 2] & 0x3f) << 6) |
  100.                       ((uint8[i + 3] & 0x3f));
  101.           i += 3;  // Consume three extra bytes.
  102.         }
  104.         // JavaScript strings are a series of UTF-16 characters.
  105.         if (codePoint <= 0xffff) {
  106.           decoded += String.fromCharCode(codePoint);
  107.         } else {
  108.           // UTF-16 surrogate-pair encoding, based on
  109.           // https://en.wikipedia.org/wiki/UTF-16#Description
  110.           const baseCodePoint = codePoint - 0x10000;
  111.           const highPart = baseCodePoint >> 10;
  112.           const lowPart = baseCodePoint & 0x3ff;
  113.           decoded += String.fromCharCode(0xd800 + highPart);
  114.           decoded += String.fromCharCode(0xdc00 + lowPart);
  115.         }
  116.       }
  118.       return decoded;
  119.     }
  120.   }
  123.   /**
  124.    * Creates a string from the given buffer as UTF-16 encoding.
  125.    *
  126.    * @param {?BufferSource} data
  127.    * @param {boolean} littleEndian
  128.          true to read little endian, false to read big.
  129.    * @param {boolean=} noThrow true to avoid throwing in cases where we may
  130.    *     expect invalid input.  If noThrow is true and the data has an odd
  131.    *     length,it will be truncated.
  132.    * @return {string}
  133.    * @export
  134.    */
  135.   static fromUTF16(data, littleEndian, noThrow) {
  136.     if (!data) {
  137.       return '';
  138.     }
  140.     if (!noThrow && data.byteLength % 2 != 0) {
  141.       shaka.log.error('Data has an incorrect length, must be even.');
  142.       throw new shaka.util.Error(
  143.           shaka.util.Error.Severity.CRITICAL, shaka.util.Error.Category.TEXT,
  144.           shaka.util.Error.Code.BAD_ENCODING);
  145.     }
  147.     // Use a DataView to ensure correct endianness.
  148.     const length = Math.floor(data.byteLength / 2);
  149.     const arr = new Uint16Array(length);
  150.     const dataView = shaka.util.BufferUtils.toDataView(data);
  151.     for (let i = 0; i < length; i++) {
  152.       arr[i] = dataView.getUint16(i * 2, littleEndian);
  153.     }
  154.     return shaka.util.StringUtils.fromCharCode(arr);
  155.   }
  158.   /**
  159.    * Creates a string from the given buffer, auto-detecting the encoding that is
  160.    * being used.  If it cannot detect the encoding, it will throw an exception.
  161.    *
  162.    * @param {?BufferSource} data
  163.    * @return {string}
  164.    * @export
  165.    */
  166.   static fromBytesAutoDetect(data) {
  167.     const StringUtils = shaka.util.StringUtils;
  168.     if (!data) {
  169.       return '';
  170.     }
  172.     const uint8 = shaka.util.BufferUtils.toUint8(data);
  173.     if (uint8[0] == 0xef && uint8[1] == 0xbb && uint8[2] == 0xbf) {
  174.       return StringUtils.fromUTF8(uint8);
  175.     } else if (uint8[0] == 0xfe && uint8[1] == 0xff) {
  176.       return StringUtils.fromUTF16(
  177.           uint8.subarray(2), /* littleEndian= */ false);
  178.     } else if (uint8[0] == 0xff && uint8[1] == 0xfe) {
  179.       return StringUtils.fromUTF16(uint8.subarray(2), /* littleEndian= */ true);
  180.     }
  182.     const isAscii = (i) => {
  183.       // arr[i] >= horizontal tab && arr[i] <= '~';
  184.       return uint8.byteLength <= i || (uint8[i] >= 0x09 && uint8[i] <= 0x7e);
  185.     };
  187.     shaka.log.debug(
  188.         'Unable to find byte-order-mark, making an educated guess.');
  189.     if (uint8[0] == 0 && uint8[2] == 0) {
  190.       return StringUtils.fromUTF16(data, /* littleEndian= */ false);
  191.     } else if (uint8[1] == 0 && uint8[3] == 0) {
  192.       return StringUtils.fromUTF16(data, /* littleEndian= */ true);
  193.     } else if (isAscii(0) && isAscii(1) && isAscii(2) && isAscii(3)) {
  194.       return StringUtils.fromUTF8(data);
  195.     }
  197.     throw new shaka.util.Error(
  198.         shaka.util.Error.Severity.CRITICAL,
  199.         shaka.util.Error.Category.TEXT,
  200.         shaka.util.Error.Code.UNABLE_TO_DETECT_ENCODING);
  201.   }
  204.   /**
  205.    * Creates a ArrayBuffer from the given string, converting to UTF-8 encoding.
  206.    *
  207.    * @param {string} str
  208.    * @return {!ArrayBuffer}
  209.    * @export
  210.    */
  211.   static toUTF8(str) {
  212.     if (window.TextEncoder && !shaka.device.DeviceFactory.getDevice()
  213.         .shouldAvoidUseTextDecoderEncoder()) {
  214.       const utf8Encoder = new TextEncoder();
  215.       return shaka.util.BufferUtils.toArrayBuffer(utf8Encoder.encode(str));
  216.     } else {
  217.       // http://stackoverflow.com/a/13691499
  218.       // Converts the given string to a URI encoded string.  If a character
  219.       // falls in the ASCII range, it is not converted; otherwise it will be
  220.       // converted to a series of URI escape sequences according to UTF-8.
  221.       // Example: 'g#€' -> 'g#%E3%82%AC'
  222.       const encoded = encodeURIComponent(str);
  223.       // Convert each escape sequence individually into a character.  Each
  224.       // escape sequence is interpreted as a code-point, so if an escape
  225.       // sequence happens to be part of a multi-byte sequence, each byte will
  226.       // be converted to a single character.
  227.       // Example: 'g#%E3%82%AC' -> '\x67\x35\xe3\x82\xac'
  228.       const utf8 = unescape(encoded);
  230.       const result = new Uint8Array(utf8.length);
  231.       for (let i = 0; i < utf8.length; i++) {
  232.         const item = utf8[i];
  233.         result[i] = item.charCodeAt(0);
  234.       }
  235.       return shaka.util.BufferUtils.toArrayBuffer(result);
  236.     }
  237.   }
  240.   /**
  241.    * Creates a ArrayBuffer from the given string, converting to UTF-16 encoding.
  242.    *
  243.    * @param {string} str
  244.    * @param {boolean} littleEndian
  245.    * @return {!ArrayBuffer}
  246.    * @export
  247.    */
  248.   static toUTF16(str, littleEndian) {
  249.     const result = new ArrayBuffer(str.length * 2);
  250.     const view = new DataView(result);
  251.     for (let i = 0; i < str.length; ++i) {
  252.       const value = str.charCodeAt(i);
  253.       view.setUint16(/* position= */ i * 2, value, littleEndian);
  254.     }
  255.     return result;
  256.   }
  259.   /**
  260.    * Creates a new string from the given array of char codes.
  261.    *
  262.    * Using String.fromCharCode.apply is risky because you can trigger stack
  263.    * errors on very large arrays.  This breaks up the array into several pieces
  264.    * to avoid this.
  265.    *
  266.    * @param {!TypedArray} array
  267.    * @return {string}
  268.    */
  269.   static fromCharCode(array) {
  270.     return shaka.util.StringUtils.fromCharCodeImpl_.value()(array);
  271.   }
  273.   /**
  274.    * Resets the fromCharCode method's implementation.
  275.    * For debug use.
  276.    * @export
  277.    */
  278.   static resetFromCharCode() {
  279.     shaka.util.StringUtils.fromCharCodeImpl_.reset();
  280.   }
  282.   /**
  283.    * This method converts the HTML entities &amp;, &lt;, &gt;, &quot;, &#39;,
  284.    * &nbsp;, &lrm; and &rlm; in string to their corresponding characters.
  285.    *
  286.    * @param {!string} input
  287.    * @return {string}
  288.    */
  289.   static htmlUnescape(input) {
  290.     // Used to map HTML entities to characters.
  291.     const htmlUnescapes = {
  292.       '&amp;': '&',
  293.       '&lt;': '<',
  294.       '&gt;': '>',
  295.       '&quot;': '"',
  296.       '&apos;': '\'',
  297.       '&nbsp;': '\u{a0}',
  298.       '&lrm;': '\u{200e}',
  299.       '&rlm;': '\u{200f}',
  300.     };
  302.     // Used to match HTML entities and HTML characters.
  303.     const reEscapedHtml =
  304.         /&(?:amp|lt|gt|quot|apos|nbsp|lrm|rlm|#[xX]?[0-9a-fA-F]+);/g;
  305.     const reHasEscapedHtml = RegExp(reEscapedHtml.source);
  306.     // This check is an optimization, since replace always makes a copy
  307.     if (input && reHasEscapedHtml.test(input)) {
  308.       return input.replace(reEscapedHtml, (entity) => {
  309.         if (entity[1] == '#') {
  310.           // Translate this into an HTML character.
  311.           let code = 0;
  312.           if (entity[2] == 'x' || entity[2] == 'X') {
  313.             // It's hex.
  314.             code = parseInt(entity.substring(3), 16);
  315.           } else {
  316.             // It's decimal.
  317.             code = parseInt(entity.substring(2), 10);
  318.           }
  319.           // Ignore it if it's an invalid code point.
  320.           if (code >= 0 && code <= 0x10FFFF) {
  321.             return String.fromCodePoint(code);
  322.           } else {
  323.             return '';
  324.           }
  325.         }
  326.         // The only thing that might not match the dictionary above is the
  327.         // single quote, which can be matched by many strings in the regex, but
  328.         // only has a single entry in the dictionary.
  329.         return htmlUnescapes[entity] || '\'';
  330.       });
  331.     }
  332.     return input || '';
  333.   }
  334. };
  337. /** @private {!shaka.util.Lazy.<function(!TypedArray):string>} */
  338. shaka.util.StringUtils.fromCharCodeImpl_ = new shaka.util.Lazy(() => {
  339.   /**
  340.    * @param {number} size
  341.    * @return {boolean}
  342.    */
  343.   const supportsChunkSize = (size) => {
  344.     try {
  345.       // The compiler will complain about suspicious value if this isn't
  346.       // stored in a variable and used.
  347.       const buffer = new Uint8Array(size);
  349.       // This can't use the spread operator, or it blows up on Xbox One.
  350.       // So we use apply() instead, which is normally not allowed.
  351.       // See issue #2186 for more details.
  352.       const foo = String.fromCharCode.apply(null, buffer);
  353.       goog.asserts.assert(foo, 'Should get value');
  354.       return foo.length > 0; // Actually use "foo", so it's not compiled out.
  355.     } catch (error) {
  356.       return false;
  357.     }
  358.   };
  360.   // Different browsers support different chunk sizes; find out the largest
  361.   // this browser supports so we can use larger chunks on supported browsers
  362.   // but still support lower-end devices that require small chunks.
  363.   // 64k is supported on all major desktop browsers.
  364.   for (let size = 64 * 1024; size > 0; size /= 2) {
  365.     if (supportsChunkSize(size)) {
  366.       return (buffer) => {
  367.         let ret = '';
  368.         for (let i = 0; i < buffer.length; i += size) {
  369.           const subArray = buffer.subarray(i, i + size);
  371.           // This can't use the spread operator, or it blows up on Xbox One.
  372.           // So we use apply() instead, which is normally not allowed.
  373.           // See issue #2186 for more details.
  374.           ret += String.fromCharCode.apply(null, subArray);  // Issue #2186
  375.         }
  376.         return ret;
  377.       };
  378.     }
  379.   }
  380.   goog.asserts.assert(false, 'Unable to create a fromCharCode method');
  381.   return null;
  382. });