Source: lib/util/ts_parser.js

  1. /*! @license
  2. * Shaka Player
  3. * Copyright 2016 Google LLC
  4. * SPDX-License-Identifier: Apache-2.0
  5. */
  6. goog.provide('shaka.util.TsParser');
  7. goog.require('goog.asserts');
  8. goog.require('shaka.Deprecate');
  9. goog.require('shaka.log');
  10. goog.require('shaka.util.ExpGolomb');
  11. goog.require('shaka.util.Id3Utils');
  12. goog.require('shaka.util.StringUtils');
  13. goog.require('shaka.util.Uint8ArrayUtils');
  14. /**
  15. * @see https://en.wikipedia.org/wiki/MPEG_transport_stream
  16. * @export
  17. */
  18. shaka.util.TsParser = class {
  19. /** */
  20. constructor() {
  21. /** @private {?number} */
  22. this.pmtId_ = null;
  23. /** @private {boolean} */
  24. this.pmtParsed_ = false;
  25. /** @private {?number} */
  26. this.videoPid_ = null;
  27. /** @private {?string} */
  28. this.videoCodec_ = null;
  29. /** @private {!Array<!Array<Uint8Array>>} */
  30. this.videoData_ = [];
  31. /** @private {!Array<shaka.extern.MPEG_PES>} */
  32. this.videoPes_ = [];
  33. /** @private {?number} */
  34. this.audioPid_ = null;
  35. /** @private {?string} */
  36. this.audioCodec_ = null;
  37. /** @private {!Array<!Array<Uint8Array>>} */
  38. this.audioData_ = [];
  39. /** @private {!Array<shaka.extern.MPEG_PES>} */
  40. this.audioPes_ = [];
  41. /** @private {?number} */
  42. this.id3Pid_ = null;
  43. /** @private {!Array<!Array<Uint8Array>>} */
  44. this.id3Data_ = [];
  45. /** @private {?number} */
  46. this.referencePts_ = null;
  47. /** @private {?number} */
  48. this.referenceDts_ = null;
  49. /** @private {?shaka.util.TsParser.OpusMetadata} */
  50. this.opusMetadata_ = null;
  51. }
  52. /**
  53. * Clear previous data
  54. *
  55. * @export
  56. */
  57. clearData() {
  58. this.videoData_ = [];
  59. this.videoPes_ = [];
  60. this.audioData_ = [];
  61. this.audioPes_ = [];
  62. this.id3Data_ = [];
  63. }
  64. /**
  65. * Parse the given data
  66. *
  67. * @param {Uint8Array} data
  68. * @return {!shaka.util.TsParser}
  69. * @export
  70. */
  71. parse(data) {
  72. const packetLength = shaka.util.TsParser.PacketLength_;
  73. // A TS fragment should contain at least 3 TS packets, a PAT, a PMT, and
  74. // one PID.
  75. if (data.length < 3 * packetLength) {
  76. return this;
  77. }
  78. const syncOffset = Math.max(0, shaka.util.TsParser.syncOffset(data));
  79. const length = data.length - (data.length + syncOffset) % packetLength;
  80. let unknownPIDs = false;
  81. // loop through TS packets
  82. for (let start = syncOffset; start < length; start += packetLength) {
  83. if (data[start] == 0x47) {
  84. const payloadUnitStartIndicator = !!(data[start + 1] & 0x40);
  85. // pid is a 13-bit field starting at the last 5 bits of TS[1]
  86. const pid = ((data[start + 1] & 0x1f) << 8) + data[start + 2];
  87. const adaptationFieldControl = (data[start + 3] & 0x30) >> 4;
  88. // if an adaption field is present, its length is specified by the
  89. // fifth byte of the TS packet header.
  90. let offset;
  91. if (adaptationFieldControl > 1) {
  92. offset = start + 5 + data[start + 4];
  93. // continue if there is only adaptation field
  94. if (offset == start + packetLength) {
  95. continue;
  96. }
  97. } else {
  98. offset = start + 4;
  99. }
  100. switch (pid) {
  101. case 0:
  102. if (payloadUnitStartIndicator) {
  103. offset += data[offset] + 1;
  104. }
  105. this.pmtId_ = this.getPmtId_(data, offset);
  106. break;
  107. case 17:
  108. case 0x1fff:
  109. break;
  110. case this.pmtId_: {
  111. if (payloadUnitStartIndicator) {
  112. offset += data[offset] + 1;
  113. }
  114. const parsedPIDs = this.parsePMT_(data, offset);
  115. // only update track id if track PID found while parsing PMT
  116. // this is to avoid resetting the PID to -1 in case
  117. // track PID transiently disappears from the stream
  118. // this could happen in case of transient missing audio samples
  119. // for example
  120. // NOTE this is only the PID of the track as found in TS,
  121. // but we are not using this for MP4 track IDs.
  122. if (parsedPIDs.video != -1) {
  123. this.videoPid_ = parsedPIDs.video;
  124. this.videoCodec_ = parsedPIDs.videoCodec;
  125. }
  126. if (parsedPIDs.audio != -1) {
  127. this.audioPid_ = parsedPIDs.audio;
  128. this.audioCodec_ = parsedPIDs.audioCodec;
  129. }
  130. if (parsedPIDs.id3 != -1) {
  131. this.id3Pid_ = parsedPIDs.id3;
  132. }
  133. if (unknownPIDs && !this.pmtParsed_) {
  134. shaka.log.debug('reparse from beginning');
  135. unknownPIDs = false;
  136. // we set it to -188, the += 188 in the for loop will reset
  137. // start to 0
  138. start = syncOffset - packetLength;
  139. }
  140. this.pmtParsed_ = true;
  141. break;
  142. }
  143. case this.videoPid_: {
  144. const videoData = data.subarray(offset, start + packetLength);
  145. if (payloadUnitStartIndicator) {
  146. this.videoData_.push([videoData]);
  147. } else if (this.videoData_.length) {
  148. const prevVideoData = this.videoData_[this.videoData_.length - 1];
  149. if (prevVideoData) {
  150. this.videoData_[this.videoData_.length - 1].push(videoData);
  151. }
  152. }
  153. break;
  154. }
  155. case this.audioPid_: {
  156. const audioData = data.subarray(offset, start + packetLength);
  157. if (payloadUnitStartIndicator) {
  158. this.audioData_.push([audioData]);
  159. } else if (this.audioData_.length) {
  160. const prevAudioData = this.audioData_[this.audioData_.length - 1];
  161. if (prevAudioData) {
  162. this.audioData_[this.audioData_.length - 1].push(audioData);
  163. }
  164. }
  165. break;
  166. }
  167. case this.id3Pid_: {
  168. const id3Data = data.subarray(offset, start + packetLength);
  169. if (payloadUnitStartIndicator) {
  170. this.id3Data_.push([id3Data]);
  171. } else if (this.id3Data_.length) {
  172. const prevId3Data = this.id3Data_[this.id3Data_.length - 1];
  173. if (prevId3Data) {
  174. this.id3Data_[this.id3Data_.length - 1].push(id3Data);
  175. }
  176. }
  177. break;
  178. }
  179. default:
  180. unknownPIDs = true;
  181. break;
  182. }
  183. } else {
  184. shaka.log.warning('Found TS packet that do not start with 0x47');
  185. }
  186. }
  187. return this;
  188. }
  189. /**
  190. * Get the PMT ID from the PAT
  191. *
  192. * @param {Uint8Array} data
  193. * @param {number} offset
  194. * @return {number}
  195. * @private
  196. */
  197. getPmtId_(data, offset) {
  198. // skip the PSI header and parse the first PMT entry
  199. return ((data[offset + 10] & 0x1f) << 8) | data[offset + 11];
  200. }
  201. /**
  202. * Parse PMT
  203. *
  204. * @param {Uint8Array} data
  205. * @param {number} offset
  206. * @return {!shaka.util.TsParser.PMT}
  207. * @private
  208. */
  209. parsePMT_(data, offset) {
  210. const StringUtils = shaka.util.StringUtils;
  211. const result = {
  212. audio: -1,
  213. video: -1,
  214. id3: -1,
  215. audioCodec: '',
  216. videoCodec: '',
  217. };
  218. const sectionLength = ((data[offset + 1] & 0x0f) << 8) | data[offset + 2];
  219. const tableEnd = offset + 3 + sectionLength - 4;
  220. // to determine where the table is, we have to figure out how
  221. // long the program info descriptors are
  222. const programInfoLength =
  223. ((data[offset + 10] & 0x0f) << 8) | data[offset + 11];
  224. // advance the offset to the first entry in the mapping table
  225. offset += 12 + programInfoLength;
  226. while (offset < tableEnd) {
  227. const pid = ((data[offset + 1] & 0x1f) << 8) | data[offset + 2];
  228. const esInfoLength = ((data[offset + 3] & 0x0f) << 8) | data[offset + 4];
  229. switch (data[offset]) {
  230. case 0x06:
  231. // stream_type 6 can mean a lot of different things in case of DVB.
  232. // We need to look at the descriptors. Right now, we're only
  233. // interested in a few audio and video formats,.
  234. if (esInfoLength > 0) {
  235. let parsePos = offset + 5;
  236. let remaining = esInfoLength;
  237. // Descriptor info: https://www.streamguru.de/mpeg-analyzer/supported-descriptor-list/
  238. while (remaining > 2) {
  239. const descriptorId = data[parsePos];
  240. const descriptorLen = data[parsePos + 1] + 2;
  241. switch (descriptorId) {
  242. // Registration descriptor
  243. case 0x05: {
  244. const registrationData =
  245. data.subarray(parsePos + 2, parsePos + descriptorLen);
  246. const registration =
  247. StringUtils.fromCharCode(registrationData);
  248. if (result.audio == -1 && registration === 'Opus') {
  249. result.audio = pid;
  250. result.audioCodec = 'opus';
  251. } else if (result.video == -1 && registration === 'AV01') {
  252. result.video = pid;
  253. result.videoCodec = 'av1';
  254. }
  255. break;
  256. }
  257. // DVB Descriptor for AC-3
  258. case 0x6a:
  259. if (result.audio == -1) {
  260. result.audio = pid;
  261. result.audioCodec = 'ac3';
  262. }
  263. break;
  264. // DVB Descriptor for EC-3
  265. case 0x7a:
  266. if (result.audio == -1) {
  267. result.audio = pid;
  268. result.audioCodec = 'ec3';
  269. }
  270. break;
  271. // DVB Descriptor for AAC
  272. case 0x7c:
  273. if (result.audio == -1) {
  274. result.audio = pid;
  275. result.audioCodec = 'aac';
  276. }
  277. break;
  278. // DVB extension descriptor
  279. case 0x7f:
  280. if (result.audioCodec == 'opus') {
  281. const extensionDescriptorId = data[parsePos + 2];
  282. let channelConfigCode = null;
  283. // User defined (provisional Opus)
  284. if (extensionDescriptorId === 0x80) {
  285. channelConfigCode = data[parsePos + 3];
  286. }
  287. if (channelConfigCode == null) {
  288. // Not Supported Opus channel count.
  289. break;
  290. }
  291. const channelCount = (channelConfigCode & 0x0F) === 0 ?
  292. 2 : (channelConfigCode & 0x0F);
  293. this.opusMetadata_ = {
  294. channelCount,
  295. channelConfigCode,
  296. sampleRate: 48000,
  297. };
  298. }
  299. break;
  300. }
  301. parsePos += descriptorLen;
  302. remaining -= descriptorLen;
  303. }
  304. }
  305. break;
  306. // SAMPLE-AES AAC
  307. case 0xcf:
  308. break;
  309. // ISO/IEC 13818-7 ADTS AAC (MPEG-2 lower bit-rate audio)
  310. case 0x0f:
  311. if (result.audio == -1) {
  312. result.audio = pid;
  313. result.audioCodec = 'aac';
  314. }
  315. break;
  316. // LOAS/LATM AAC
  317. case 0x11:
  318. if (result.audio == -1) {
  319. result.audio = pid;
  320. result.audioCodec = 'aac-loas';
  321. }
  322. break;
  323. // Packetized metadata (ID3)
  324. case 0x15:
  325. if (result.id3 == -1) {
  326. result.id3 = pid;
  327. }
  328. break;
  329. // SAMPLE-AES AVC
  330. case 0xdb:
  331. break;
  332. // ITU-T Rec. H.264 and ISO/IEC 14496-10 (lower bit-rate video)
  333. case 0x1b:
  334. if (result.video == -1) {
  335. result.video = pid;
  336. result.videoCodec = 'avc';
  337. }
  338. break;
  339. // ISO/IEC 11172-3 (MPEG-1 audio)
  340. // or ISO/IEC 13818-3 (MPEG-2 halved sample rate audio)
  341. case 0x03:
  342. case 0x04:
  343. if (result.audio == -1) {
  344. result.audio = pid;
  345. result.audioCodec = 'mp3';
  346. }
  347. break;
  348. // HEVC
  349. case 0x24:
  350. if (result.video == -1) {
  351. result.video = pid;
  352. result.videoCodec = 'hvc';
  353. }
  354. break;
  355. // AC-3
  356. case 0x81:
  357. if (result.audio == -1) {
  358. result.audio = pid;
  359. result.audioCodec = 'ac3';
  360. }
  361. break;
  362. // EC-3
  363. case 0x84:
  364. case 0x87:
  365. if (result.audio == -1) {
  366. result.audio = pid;
  367. result.audioCodec = 'ec3';
  368. }
  369. break;
  370. default:
  371. // shaka.log.warning('Unknown stream type:', data[offset]);
  372. break;
  373. }
  374. // move to the next table entry
  375. // skip past the elementary stream descriptors, if present
  376. offset += esInfoLength + 5;
  377. }
  378. return result;
  379. }
  380. /**
  381. * Parse PES
  382. *
  383. * @param {Uint8Array} data
  384. * @return {?shaka.extern.MPEG_PES}
  385. * @private
  386. */
  387. parsePES_(data) {
  388. const startPrefix = (data[0] << 16) | (data[1] << 8) | data[2];
  389. // In certain live streams, the start of a TS fragment has ts packets
  390. // that are frame data that is continuing from the previous fragment. This
  391. // is to check that the pes data is the start of a new pes data
  392. if (startPrefix !== 1) {
  393. return null;
  394. }
  395. /** @type {shaka.extern.MPEG_PES} */
  396. const pes = {
  397. data: new Uint8Array(0),
  398. // get the packet length, this will be 0 for video
  399. packetLength: ((data[4] << 8) | data[5]),
  400. pts: null,
  401. dts: null,
  402. nalus: [],
  403. };
  404. // if PES parsed length is not zero and greater than total received length,
  405. // stop parsing. PES might be truncated. minus 6 : PES header size
  406. if (pes.packetLength && pes.packetLength > data.byteLength - 6) {
  407. return null;
  408. }
  409. // PES packets may be annotated with a PTS value, or a PTS value
  410. // and a DTS value. Determine what combination of values is
  411. // available to work with.
  412. const ptsDtsFlags = data[7];
  413. // PTS and DTS are normally stored as a 33-bit number. Javascript
  414. // performs all bitwise operations on 32-bit integers but javascript
  415. // supports a much greater range (52-bits) of integer using standard
  416. // mathematical operations.
  417. // We construct a 31-bit value using bitwise operators over the 31
  418. // most significant bits and then multiply by 4 (equal to a left-shift
  419. // of 2) before we add the final 2 least significant bits of the
  420. // timestamp (equal to an OR.)
  421. if (ptsDtsFlags & 0xC0) {
  422. // the PTS and DTS are not written out directly. For information
  423. // on how they are encoded, see
  424. // http://dvd.sourceforge.net/dvdinfo/pes-hdr.html
  425. const pts =
  426. (data[9] & 0x0e) * 536870912 + // 1 << 29
  427. (data[10] & 0xff) * 4194304 + // 1 << 22
  428. (data[11] & 0xfe) * 16384 + // 1 << 14
  429. (data[12] & 0xff) * 128 + // 1 << 7
  430. (data[13] & 0xfe) / 2;
  431. if (this.referencePts_ == null) {
  432. this.referencePts_ = pts;
  433. }
  434. pes.pts = this.handleRollover_(pts, this.referencePts_);
  435. this.referencePts_ = pes.pts;
  436. pes.dts = pes.pts;
  437. if (ptsDtsFlags & 0x40) {
  438. const dts =
  439. (data[14] & 0x0e) * 536870912 + // 1 << 29
  440. (data[15] & 0xff) * 4194304 + // 1 << 22
  441. (data[16] & 0xfe) * 16384 + // 1 << 14
  442. (data[17] & 0xff) * 128 + // 1 << 7
  443. (data[18] & 0xfe) / 2;
  444. if (this.referenceDts_ == null) {
  445. this.referenceDts_ = dts;
  446. }
  447. if (pes.pts != pts) {
  448. pes.dts = this.handleRollover_(dts, this.referenceDts_);
  449. } else {
  450. pes.dts = dts;
  451. }
  452. }
  453. this.referenceDts_ = pes.dts;
  454. }
  455. const pesHdrLen = data[8];
  456. // 9 bytes : 6 bytes for PES header + 3 bytes for PES extension
  457. const payloadStartOffset = pesHdrLen + 9;
  458. if (data.byteLength <= payloadStartOffset) {
  459. return null;
  460. }
  461. pes.data = data.subarray(payloadStartOffset);
  462. return pes;
  463. }
  464. /**
  465. * Parse AVC Nalus
  466. *
  467. * The code is based on hls.js
  468. * Credit to https://github.com/video-dev/hls.js/blob/master/src/demux/tsdemuxer.ts
  469. *
  470. * @param {shaka.extern.MPEG_PES} pes
  471. * @param {?shaka.extern.MPEG_PES=} nextPes
  472. * @return {!Array<shaka.extern.VideoNalu>}
  473. * @export
  474. */
  475. parseAvcNalus(pes, nextPes) {
  476. shaka.Deprecate.deprecateFeature(5,
  477. 'TsParser.parseAvcNalus',
  478. 'Please use parseNalus function instead.');
  479. const lastInfo = {
  480. nalu: null,
  481. state: null,
  482. };
  483. return this.parseNalus(pes, lastInfo);
  484. }
  485. /**
  486. * Parse AVC and HVC Nalus
  487. *
  488. * The code is based on hls.js
  489. * Credit to https://github.com/video-dev/hls.js/blob/master/src/demux/tsdemuxer.ts
  490. *
  491. * @param {shaka.extern.MPEG_PES} pes
  492. * @param {{nalu: ?shaka.extern.VideoNalu, state: ?number}} lastInfo
  493. * @return {!Array<shaka.extern.VideoNalu>}
  494. * @export
  495. */
  496. parseNalus(pes, lastInfo) {
  497. const timescale = shaka.util.TsParser.Timescale;
  498. const time = pes.pts ? pes.pts / timescale : null;
  499. const data = pes.data;
  500. const len = data.byteLength;
  501. let naluHeaderSize = 1;
  502. if (this.videoCodec_ == 'hvc') {
  503. naluHeaderSize = 2;
  504. }
  505. // A NALU does not contain is its size.
  506. // The Annex B specification solves this by requiring ‘Start Codes’ to
  507. // precede each NALU. A start code is 2 or 3 0x00 bytes followed with a
  508. // 0x01 byte. e.g. 0x000001 or 0x00000001.
  509. // More info in: https://stackoverflow.com/questions/24884827/possible-locations-for-sequence-picture-parameter-sets-for-h-264-stream/24890903#24890903
  510. let numZeros = lastInfo.state || 0;
  511. const initialNumZeros = numZeros;
  512. /** @type {number} */
  513. let i = 0;
  514. /** @type {!Array<shaka.extern.VideoNalu>} */
  515. const nalus = [];
  516. // Start position includes the first byte where we read the type.
  517. // The data we extract begins at the next byte.
  518. let lastNaluStart = -1;
  519. // Extracted from the first byte.
  520. let lastNaluType = 0;
  521. const getNaluType = (offset) => {
  522. if (this.videoCodec_ == 'hvc') {
  523. return (data[offset] >> 1) & 0x3f;
  524. } else {
  525. return data[offset] & 0x1f;
  526. }
  527. };
  528. const getLastNalu = () => {
  529. if (nalus.length) {
  530. return nalus[nalus.length - 1];
  531. }
  532. return lastInfo.nalu;
  533. };
  534. if (numZeros == -1) {
  535. // special use case where we found 3 or 4-byte start codes exactly at the
  536. // end of previous PES packet
  537. lastNaluStart = 0;
  538. // NALu type is value read from offset 0
  539. lastNaluType = getNaluType(0);
  540. numZeros = 0;
  541. i = 1;
  542. }
  543. while (i < len) {
  544. const value = data[i++];
  545. // Optimization. numZeros 0 and 1 are the predominant case.
  546. if (!numZeros) {
  547. numZeros = value ? 0 : 1;
  548. continue;
  549. }
  550. if (numZeros === 1) {
  551. numZeros = value ? 0 : 2;
  552. continue;
  553. }
  554. if (!value) {
  555. numZeros = 3;
  556. } else if (value == 1) {
  557. const overflow = i - numZeros - 1;
  558. if (lastNaluStart >= 0) {
  559. /** @type {shaka.extern.VideoNalu} */
  560. const nalu = {
  561. data: data.subarray(lastNaluStart + naluHeaderSize, overflow),
  562. fullData: data.subarray(lastNaluStart, overflow),
  563. type: lastNaluType,
  564. time: time,
  565. state: null,
  566. };
  567. nalus.push(nalu);
  568. } else {
  569. const lastNalu = getLastNalu();
  570. if (lastNalu) {
  571. if (initialNumZeros && i <= 4 - initialNumZeros) {
  572. // Start delimiter overlapping between PES packets
  573. // strip start delimiter bytes from the end of last NAL unit
  574. // check if lastNalu had a state different from zero
  575. if (lastNalu.state) {
  576. // strip last bytes
  577. lastNalu.data = lastNalu.data.subarray(
  578. 0, lastNalu.data.byteLength - initialNumZeros);
  579. lastNalu.fullData = lastNalu.fullData.subarray(
  580. 0, lastNalu.fullData.byteLength - initialNumZeros);
  581. }
  582. }
  583. // If NAL units are not starting right at the beginning of the PES
  584. // packet, push preceding data into previous NAL unit.
  585. if (overflow > 0) {
  586. const prevData = data.subarray(0, overflow);
  587. lastNalu.data = shaka.util.Uint8ArrayUtils.concat(
  588. lastNalu.data, prevData);
  589. lastNalu.fullData = shaka.util.Uint8ArrayUtils.concat(
  590. lastNalu.fullData, prevData);
  591. lastNalu.state = 0;
  592. }
  593. }
  594. }
  595. // Check if we can read unit type
  596. if (i < len) {
  597. lastNaluType = getNaluType(i);
  598. lastNaluStart = i;
  599. numZeros = 0;
  600. } else {
  601. // Not enough byte to read unit type.
  602. // Let's read it on next PES parsing.
  603. numZeros = -1;
  604. }
  605. } else {
  606. numZeros = 0;
  607. }
  608. }
  609. if (lastNaluStart >= 0 && numZeros >= 0) {
  610. const nalu = {
  611. data: data.subarray(lastNaluStart + naluHeaderSize, len),
  612. fullData: data.subarray(lastNaluStart, len),
  613. type: lastNaluType,
  614. time: time,
  615. state: numZeros,
  616. };
  617. nalus.push(nalu);
  618. }
  619. if (!nalus.length && lastInfo.nalu) {
  620. const lastNalu = getLastNalu();
  621. if (lastNalu) {
  622. lastNalu.data = shaka.util.Uint8ArrayUtils.concat(
  623. lastNalu.data, data);
  624. lastNalu.fullData = shaka.util.Uint8ArrayUtils.concat(
  625. lastNalu.fullData, data);
  626. }
  627. }
  628. lastInfo.state = numZeros;
  629. return nalus;
  630. }
  631. /**
  632. * Return the ID3 metadata
  633. *
  634. * @return {!Array<shaka.extern.ID3Metadata>}
  635. * @export
  636. */
  637. getMetadata() {
  638. const timescale = shaka.util.TsParser.Timescale;
  639. const metadata = [];
  640. for (const id3DataArray of this.id3Data_) {
  641. const id3Data = shaka.util.Uint8ArrayUtils.concat(...id3DataArray);
  642. const pes = this.parsePES_(id3Data);
  643. if (pes) {
  644. metadata.push({
  645. cueTime: pes.pts ? pes.pts / timescale : null,
  646. data: pes.data,
  647. frames: shaka.util.Id3Utils.getID3Frames(pes.data),
  648. dts: pes.dts,
  649. pts: pes.pts,
  650. });
  651. }
  652. }
  653. return metadata;
  654. }
  655. /**
  656. * Return the audio data
  657. *
  658. * @return {!Array<shaka.extern.MPEG_PES>}
  659. * @export
  660. */
  661. getAudioData() {
  662. if (this.audioData_.length && !this.audioPes_.length) {
  663. let sort = false;
  664. for (const audioDataArray of this.audioData_) {
  665. const audioData = shaka.util.Uint8ArrayUtils.concat(...audioDataArray);
  666. const pes = this.parsePES_(audioData);
  667. let previousPes = this.audioPes_.length ?
  668. this.audioPes_[this.audioPes_.length - 1] : null;
  669. if (pes && pes.pts != null && pes.dts != null && (!previousPes ||
  670. (previousPes.pts != pes.pts && previousPes.dts != pes.dts))) {
  671. if (this.audioPes_.length && pes.dts < (previousPes.dts || 0)) {
  672. sort = true;
  673. }
  674. this.audioPes_.push(pes);
  675. } else if (this.audioPes_.length) {
  676. const data = pes ? pes.data : audioData;
  677. if (!data) {
  678. continue;
  679. }
  680. previousPes = this.audioPes_.pop();
  681. previousPes.data =
  682. shaka.util.Uint8ArrayUtils.concat(previousPes.data, data);
  683. this.audioPes_.push(previousPes);
  684. }
  685. }
  686. if (sort) {
  687. this.audioPes_ = this.audioPes_.sort((a, b) => {
  688. const deltaDts = (a.dts || 0) - (b.dts || 0);
  689. const deltaPts = (a.pts || 0) - (b.pts || 0);
  690. return deltaDts || deltaPts;
  691. });
  692. }
  693. }
  694. return this.audioPes_;
  695. }
  696. /**
  697. * Return the video data
  698. *
  699. * @param {boolean=} naluProcessing
  700. * @return {!Array<shaka.extern.MPEG_PES>}
  701. * @export
  702. */
  703. getVideoData(naluProcessing = true) {
  704. if (this.videoData_.length && !this.videoPes_.length) {
  705. let sort = false;
  706. for (const videoDataArray of this.videoData_) {
  707. const videoData = shaka.util.Uint8ArrayUtils.concat(...videoDataArray);
  708. const pes = this.parsePES_(videoData);
  709. let previousPes = this.videoPes_.length ?
  710. this.videoPes_[this.videoPes_.length - 1] : null;
  711. if (pes && pes.pts != null && pes.dts != null && (!previousPes ||
  712. (previousPes.pts != pes.pts && previousPes.dts != pes.dts))) {
  713. if (previousPes && this.videoPes_.length &&
  714. pes.dts < (previousPes.dts || pes.dts) &&
  715. pes.pts < (previousPes.pts || pes.pts)) {
  716. sort = true;
  717. }
  718. this.videoPes_.push(pes);
  719. } else if (this.videoPes_.length) {
  720. const data = pes ? pes.data : videoData;
  721. if (!data) {
  722. continue;
  723. }
  724. previousPes = this.videoPes_.pop();
  725. previousPes.data =
  726. shaka.util.Uint8ArrayUtils.concat(previousPes.data, data);
  727. this.videoPes_.push(previousPes);
  728. }
  729. }
  730. if (naluProcessing) {
  731. const lastInfo = {
  732. nalu: null,
  733. state: null,
  734. };
  735. const pesWithLength = [];
  736. for (const pes of this.videoPes_) {
  737. pes.nalus = this.parseNalus(pes, lastInfo);
  738. if (pes.nalus.length) {
  739. pesWithLength.push(pes);
  740. lastInfo.nalu = pes.nalus[pes.nalus.length - 1];
  741. }
  742. }
  743. this.videoPes_ = pesWithLength;
  744. }
  745. if (sort) {
  746. this.videoPes_ = this.videoPes_.sort((a, b) => {
  747. const deltaDts = (a.dts || 0) - (b.dts || 0);
  748. const deltaPts = (a.pts || 0) - (b.pts || 0);
  749. return deltaDts || deltaPts;
  750. });
  751. }
  752. }
  753. if (!naluProcessing) {
  754. const prevVideoPes = this.videoPes_;
  755. this.videoPes_ = [];
  756. return prevVideoPes;
  757. }
  758. return this.videoPes_;
  759. }
  760. /**
  761. * Return the start time for the audio and video
  762. *
  763. * @param {string} contentType
  764. * @return {?number}
  765. * @export
  766. */
  767. getStartTime(contentType) {
  768. const timescale = shaka.util.TsParser.Timescale;
  769. if (contentType == 'audio') {
  770. let audioStartTime = null;
  771. const audioData = this.getAudioData();
  772. if (audioData.length) {
  773. const pes = audioData[0];
  774. audioStartTime = Math.min(pes.dts, pes.pts) / timescale;
  775. }
  776. return audioStartTime;
  777. } else if (contentType == 'video') {
  778. let videoStartTime = null;
  779. const videoData = this.getVideoData(/* naluProcessing= */ false);
  780. if (videoData.length) {
  781. const pes = videoData[0];
  782. videoStartTime = Math.min(pes.dts, pes.pts) / timescale;
  783. }
  784. return videoStartTime;
  785. }
  786. return null;
  787. }
  788. /**
  789. * Return the audio and video codecs
  790. *
  791. * @return {{audio: ?string, video: ?string}}
  792. * @export
  793. */
  794. getCodecs() {
  795. return {
  796. audio: this.audioCodec_,
  797. video: this.videoCodec_,
  798. };
  799. }
  800. /**
  801. * Return the video data
  802. *
  803. * @return {!Array<shaka.extern.VideoNalu>}
  804. * @export
  805. */
  806. getVideoNalus() {
  807. const nalus = [];
  808. for (const pes of this.getVideoData()) {
  809. nalus.push(...pes.nalus);
  810. }
  811. return nalus;
  812. }
  813. /**
  814. * Return the video resolution
  815. *
  816. * @return {{height: ?string, width: ?string}}
  817. * @export
  818. */
  819. getVideoResolution() {
  820. shaka.Deprecate.deprecateFeature(5,
  821. 'TsParser.getVideoResolution',
  822. 'Please use getVideoInfo function instead.');
  823. const videoInfo = this.getVideoInfo();
  824. return {
  825. height: videoInfo.height,
  826. width: videoInfo.width,
  827. };
  828. }
  829. /**
  830. * Return the video information
  831. *
  832. * @return {{height: ?string, width: ?string, codec: ?string,
  833. * frameRate: ?string}}
  834. * @export
  835. */
  836. getVideoInfo() {
  837. if (this.videoCodec_ == 'hvc') {
  838. return this.getHvcInfo_();
  839. }
  840. return this.getAvcInfo_();
  841. }
  842. /**
  843. * @return {?string}
  844. * @private
  845. */
  846. getFrameRate_() {
  847. const timescale = shaka.util.TsParser.Timescale;
  848. const videoData = this.getVideoData();
  849. if (videoData.length > 1) {
  850. const firstPts = videoData[0].pts;
  851. goog.asserts.assert(typeof(firstPts) == 'number',
  852. 'Should be an number!');
  853. const secondPts = videoData[1].pts;
  854. goog.asserts.assert(typeof(secondPts) == 'number',
  855. 'Should be an number!');
  856. if (!isNaN(secondPts - firstPts)) {
  857. return String(1 / (secondPts - firstPts) * timescale);
  858. }
  859. }
  860. return null;
  861. }
  862. /**
  863. * Return the video information for AVC
  864. *
  865. * @return {{height: ?string, width: ?string, codec: ?string,
  866. * frameRate: ?string}}
  867. * @private
  868. */
  869. getAvcInfo_() {
  870. const TsParser = shaka.util.TsParser;
  871. const videoInfo = {
  872. height: null,
  873. width: null,
  874. codec: null,
  875. frameRate: null,
  876. };
  877. const videoNalus = this.getVideoNalus();
  878. if (!videoNalus.length) {
  879. return videoInfo;
  880. }
  881. const spsNalu = videoNalus.find((nalu) => {
  882. return nalu.type == TsParser.H264_NALU_TYPE_SPS_;
  883. });
  884. if (!spsNalu) {
  885. return videoInfo;
  886. }
  887. const expGolombDecoder = new shaka.util.ExpGolomb(spsNalu.data);
  888. // profile_idc
  889. const profileIdc = expGolombDecoder.readUnsignedByte();
  890. // constraint_set[0-5]_flag
  891. const profileCompatibility = expGolombDecoder.readUnsignedByte();
  892. // level_idc u(8)
  893. const levelIdc = expGolombDecoder.readUnsignedByte();
  894. // seq_parameter_set_id
  895. expGolombDecoder.skipExpGolomb();
  896. // some profiles have more optional data we don't need
  897. if (TsParser.PROFILES_WITH_OPTIONAL_SPS_DATA_.includes(profileIdc)) {
  898. const chromaFormatIdc = expGolombDecoder.readUnsignedExpGolomb();
  899. if (chromaFormatIdc === 3) {
  900. // separate_colour_plane_flag
  901. expGolombDecoder.skipBits(1);
  902. }
  903. // bit_depth_luma_minus8
  904. expGolombDecoder.skipExpGolomb();
  905. // bit_depth_chroma_minus8
  906. expGolombDecoder.skipExpGolomb();
  907. // qpprime_y_zero_transform_bypass_flag
  908. expGolombDecoder.skipBits(1);
  909. // seq_scaling_matrix_present_flag
  910. if (expGolombDecoder.readBoolean()) {
  911. const scalingListCount = (chromaFormatIdc !== 3) ? 8 : 12;
  912. for (let i = 0; i < scalingListCount; i++) {
  913. // seq_scaling_list_present_flag[ i ]
  914. if (expGolombDecoder.readBoolean()) {
  915. if (i < 6) {
  916. expGolombDecoder.skipScalingList(16);
  917. } else {
  918. expGolombDecoder.skipScalingList(64);
  919. }
  920. }
  921. }
  922. }
  923. }
  924. // log2_max_frame_num_minus4
  925. expGolombDecoder.skipExpGolomb();
  926. const picOrderCntType = expGolombDecoder.readUnsignedExpGolomb();
  927. if (picOrderCntType === 0) {
  928. // log2_max_pic_order_cnt_lsb_minus4
  929. expGolombDecoder.readUnsignedExpGolomb();
  930. } else if (picOrderCntType === 1) {
  931. // delta_pic_order_always_zero_flag
  932. expGolombDecoder.skipBits(1);
  933. // offset_for_non_ref_pic
  934. expGolombDecoder.skipExpGolomb();
  935. // offset_for_top_to_bottom_field
  936. expGolombDecoder.skipExpGolomb();
  937. const numRefFramesInPicOrderCntCycle =
  938. expGolombDecoder.readUnsignedExpGolomb();
  939. for (let i = 0; i < numRefFramesInPicOrderCntCycle; i++) {
  940. // offset_for_ref_frame[ i ]
  941. expGolombDecoder.skipExpGolomb();
  942. }
  943. }
  944. // max_num_ref_frames
  945. expGolombDecoder.skipExpGolomb();
  946. // gaps_in_frame_num_value_allowed_flag
  947. expGolombDecoder.skipBits(1);
  948. const picWidthInMbsMinus1 =
  949. expGolombDecoder.readUnsignedExpGolomb();
  950. const picHeightInMapUnitsMinus1 =
  951. expGolombDecoder.readUnsignedExpGolomb();
  952. const frameMbsOnlyFlag = expGolombDecoder.readBits(1);
  953. if (frameMbsOnlyFlag === 0) {
  954. // mb_adaptive_frame_field_flag
  955. expGolombDecoder.skipBits(1);
  956. }
  957. // direct_8x8_inference_flag
  958. expGolombDecoder.skipBits(1);
  959. let frameCropLeftOffset = 0;
  960. let frameCropRightOffset = 0;
  961. let frameCropTopOffset = 0;
  962. let frameCropBottomOffset = 0;
  963. // frame_cropping_flag
  964. if (expGolombDecoder.readBoolean()) {
  965. frameCropLeftOffset = expGolombDecoder.readUnsignedExpGolomb();
  966. frameCropRightOffset = expGolombDecoder.readUnsignedExpGolomb();
  967. frameCropTopOffset = expGolombDecoder.readUnsignedExpGolomb();
  968. frameCropBottomOffset = expGolombDecoder.readUnsignedExpGolomb();
  969. }
  970. videoInfo.height = String(((2 - frameMbsOnlyFlag) *
  971. (picHeightInMapUnitsMinus1 + 1) * 16) - (frameCropTopOffset * 2) -
  972. (frameCropBottomOffset * 2));
  973. videoInfo.width = String(((picWidthInMbsMinus1 + 1) * 16) -
  974. frameCropLeftOffset * 2 - frameCropRightOffset * 2);
  975. videoInfo.codec = 'avc1.' + this.byteToHex_(profileIdc) +
  976. this.byteToHex_(profileCompatibility) + this.byteToHex_(levelIdc);
  977. videoInfo.frameRate = this.getFrameRate_();
  978. return videoInfo;
  979. }
  980. /**
  981. * Return the video information for HVC
  982. *
  983. * @return {{height: ?string, width: ?string, codec: ?string,
  984. * frameRate: ?string}}
  985. * @private
  986. */
  987. getHvcInfo_() {
  988. const TsParser = shaka.util.TsParser;
  989. const videoInfo = {
  990. height: null,
  991. width: null,
  992. codec: null,
  993. frameRate: null,
  994. };
  995. const videoNalus = this.getVideoNalus();
  996. if (!videoNalus.length) {
  997. return videoInfo;
  998. }
  999. const spsNalu = videoNalus.find((nalu) => {
  1000. return nalu.type == TsParser.H265_NALU_TYPE_SPS_;
  1001. });
  1002. if (!spsNalu) {
  1003. return videoInfo;
  1004. }
  1005. const gb = new shaka.util.ExpGolomb(
  1006. spsNalu.fullData, /* convertEbsp2rbsp= */ true);
  1007. // remove NALu Header
  1008. gb.readUnsignedByte();
  1009. gb.readUnsignedByte();
  1010. // SPS
  1011. gb.readBits(4); // video_parameter_set_id
  1012. const maxSubLayersMinus1 = gb.readBits(3);
  1013. gb.readBoolean(); // temporal_id_nesting_flag
  1014. // profile_tier_level begin
  1015. const generalProfileSpace = gb.readBits(2);
  1016. const generalTierFlag = gb.readBits(1);
  1017. const generalProfileIdc = gb.readBits(5);
  1018. const generalProfileCompatibilityFlags = gb.readBits(32);
  1019. const generalConstraintIndicatorFlags1 = gb.readUnsignedByte();
  1020. const generalConstraintIndicatorFlags2 = gb.readUnsignedByte();
  1021. const generalConstraintIndicatorFlags3 = gb.readUnsignedByte();
  1022. const generalConstraintIndicatorFlags4 = gb.readUnsignedByte();
  1023. const generalConstraintIndicatorFlags5 = gb.readUnsignedByte();
  1024. const generalConstraintIndicatorFlags6 = gb.readUnsignedByte();
  1025. const generalLevelIdc = gb.readUnsignedByte();
  1026. const subLayerProfilePresentFlag = [];
  1027. const subLayerLevelPresentFlag = [];
  1028. for (let i = 0; i < maxSubLayersMinus1; i++) {
  1029. subLayerProfilePresentFlag.push(gb.readBoolean());
  1030. subLayerLevelPresentFlag.push(gb.readBoolean());
  1031. }
  1032. if (maxSubLayersMinus1 > 0) {
  1033. for (let i = maxSubLayersMinus1; i < 8; i++) {
  1034. gb.readBits(2);
  1035. }
  1036. }
  1037. for (let i = 0; i < maxSubLayersMinus1; i++) {
  1038. if (subLayerProfilePresentFlag[i]) {
  1039. gb.readBits(88);
  1040. }
  1041. if (subLayerLevelPresentFlag[i]) {
  1042. gb.readUnsignedByte();
  1043. }
  1044. }
  1045. // profile_tier_level end
  1046. gb.readUnsignedExpGolomb(); // seq_parameter_set_id
  1047. const chromaFormatIdc = gb.readUnsignedExpGolomb();
  1048. if (chromaFormatIdc == 3) {
  1049. gb.readBits(1); // separate_colour_plane_flag
  1050. }
  1051. const picWidthInLumaSamples = gb.readUnsignedExpGolomb();
  1052. const picHeightInLumaSamples = gb.readUnsignedExpGolomb();
  1053. let leftOffset = 0;
  1054. let rightOffset = 0;
  1055. let topOffset = 0;
  1056. let bottomOffset = 0;
  1057. const conformanceWindowFlag = gb.readBoolean();
  1058. if (conformanceWindowFlag) {
  1059. leftOffset += gb.readUnsignedExpGolomb();
  1060. rightOffset += gb.readUnsignedExpGolomb();
  1061. topOffset += gb.readUnsignedExpGolomb();
  1062. bottomOffset += gb.readUnsignedExpGolomb();
  1063. }
  1064. const subWc = chromaFormatIdc === 1 || chromaFormatIdc === 2 ? 2 : 1;
  1065. const subHc = chromaFormatIdc === 1 ? 2 : 1;
  1066. videoInfo.width =
  1067. String(picWidthInLumaSamples - (leftOffset + rightOffset) * subWc);
  1068. videoInfo.height =
  1069. String(picHeightInLumaSamples - (topOffset + bottomOffset) * subHc);
  1070. const reverseBits = (integer) => {
  1071. let result = 0;
  1072. for (let i = 0; i < 32; i++) {
  1073. result |= ((integer >> i) & 1) << (31 - i);
  1074. }
  1075. return result >>> 0;
  1076. };
  1077. const profileSpace = ['', 'A', 'B', 'C'][generalProfileSpace];
  1078. const profileCompatibility = reverseBits(generalProfileCompatibilityFlags);
  1079. const tierFlag = generalTierFlag == 1 ? 'H' : 'L';
  1080. let codec = 'hvc1';
  1081. codec += '.' + profileSpace + generalProfileIdc;
  1082. codec += '.' + profileCompatibility.toString(16).toUpperCase();
  1083. codec += '.' + tierFlag + generalLevelIdc;
  1084. if (generalConstraintIndicatorFlags6) {
  1085. codec += '.' +
  1086. generalConstraintIndicatorFlags6.toString(16).toUpperCase();
  1087. }
  1088. if (generalConstraintIndicatorFlags5) {
  1089. codec += '.' +
  1090. generalConstraintIndicatorFlags5.toString(16).toUpperCase();
  1091. }
  1092. if (generalConstraintIndicatorFlags4) {
  1093. codec += '.' +
  1094. generalConstraintIndicatorFlags4.toString(16).toUpperCase();
  1095. }
  1096. if (generalConstraintIndicatorFlags3) {
  1097. codec += '.' +
  1098. generalConstraintIndicatorFlags3.toString(16).toUpperCase();
  1099. }
  1100. if (generalConstraintIndicatorFlags2) {
  1101. codec += '.' +
  1102. generalConstraintIndicatorFlags2.toString(16).toUpperCase();
  1103. }
  1104. if (generalConstraintIndicatorFlags1) {
  1105. codec += '.' +
  1106. generalConstraintIndicatorFlags1.toString(16).toUpperCase();
  1107. }
  1108. videoInfo.codec = codec;
  1109. videoInfo.frameRate = this.getFrameRate_();
  1110. return videoInfo;
  1111. }
  1112. /**
  1113. * Return the Opus metadata
  1114. *
  1115. * @return {?shaka.util.TsParser.OpusMetadata}
  1116. */
  1117. getOpusMetadata() {
  1118. return this.opusMetadata_;
  1119. }
  1120. /**
  1121. * Convert a byte to 2 digits of hex. (Only handles values 0-255.)
  1122. *
  1123. * @param {number} x
  1124. * @return {string}
  1125. * @private
  1126. */
  1127. byteToHex_(x) {
  1128. return ('0' + x.toString(16).toUpperCase()).slice(-2);
  1129. }
  1130. /**
  1131. * @param {number} value
  1132. * @param {number} reference
  1133. * @return {number}
  1134. * @private
  1135. */
  1136. handleRollover_(value, reference) {
  1137. const MAX_TS = 8589934592;
  1138. const RO_THRESH = 4294967296;
  1139. let direction = 1;
  1140. if (value > reference) {
  1141. // If the current timestamp value is greater than our reference timestamp
  1142. // and we detect a timestamp rollover, this means the roll over is
  1143. // happening in the opposite direction.
  1144. // Example scenario: Enter a long stream/video just after a rollover
  1145. // occurred. The reference point will be set to a small number, e.g. 1.
  1146. // The user then seeks backwards over the rollover point. In loading this
  1147. // segment, the timestamp values will be very large, e.g. 2^33 - 1. Since
  1148. // this comes before the data we loaded previously, we want to adjust the
  1149. // time stamp to be `value - 2^33`.
  1150. direction = -1;
  1151. }
  1152. // Note: A seek forwards or back that is greater than the RO_THRESH
  1153. // (2^32, ~13 hours) will cause an incorrect adjustment.
  1154. while (Math.abs(reference - value) > RO_THRESH) {
  1155. value += (direction * MAX_TS);
  1156. }
  1157. return value;
  1158. }
  1159. /**
  1160. * Check if the passed data corresponds to an MPEG2-TS
  1161. *
  1162. * @param {Uint8Array} data
  1163. * @return {boolean}
  1164. * @export
  1165. */
  1166. static probe(data) {
  1167. const syncOffset = shaka.util.TsParser.syncOffset(data);
  1168. if (syncOffset < 0) {
  1169. return false;
  1170. } else {
  1171. if (syncOffset > 0) {
  1172. shaka.log.warning('MPEG2-TS detected but first sync word found @ ' +
  1173. 'offset ' + syncOffset + ', junk ahead ?');
  1174. }
  1175. return true;
  1176. }
  1177. }
  1178. /**
  1179. * Returns the synchronization offset
  1180. *
  1181. * @param {Uint8Array} data
  1182. * @return {number}
  1183. * @export
  1184. */
  1185. static syncOffset(data) {
  1186. const packetLength = shaka.util.TsParser.PacketLength_;
  1187. // scan 1000 first bytes
  1188. const scanWindow = Math.min(1000, data.length - 3 * packetLength);
  1189. let i = 0;
  1190. while (i < scanWindow) {
  1191. // a TS fragment should contain at least 3 TS packets, a PAT, a PMT, and
  1192. // one PID, each starting with 0x47
  1193. if (data[i] == 0x47 &&
  1194. data[i + packetLength] == 0x47 &&
  1195. data[i + 2 * packetLength] == 0x47) {
  1196. return i;
  1197. } else {
  1198. i++;
  1199. }
  1200. }
  1201. return -1;
  1202. }
  1203. };
  1204. /**
  1205. * @const {number}
  1206. * @export
  1207. */
  1208. shaka.util.TsParser.Timescale = 90000;
  1209. /**
  1210. * @const {number}
  1211. * @private
  1212. */
  1213. shaka.util.TsParser.PacketLength_ = 188;
  1214. /**
  1215. * NALU type for Sequence Parameter Set (SPS) for H.264.
  1216. * @const {number}
  1217. * @private
  1218. */
  1219. shaka.util.TsParser.H264_NALU_TYPE_SPS_ = 0x07;
  1220. /**
  1221. * NALU type for Sequence Parameter Set (SPS) for H.265.
  1222. * @const {number}
  1223. * @private
  1224. */
  1225. shaka.util.TsParser.H265_NALU_TYPE_SPS_ = 0x21;
  1226. /**
  1227. * Values of profile_idc that indicate additional fields are included in the
  1228. * SPS.
  1229. * see Recommendation ITU-T H.264 (4/2013)
  1230. * 7.3.2.1.1 Sequence parameter set data syntax
  1231. *
  1232. * @const {!Array<number>}
  1233. * @private
  1234. */
  1235. shaka.util.TsParser.PROFILES_WITH_OPTIONAL_SPS_DATA_ =
  1236. [100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134];
  1237. /**
  1238. * @typedef {{
  1239. * audio: number,
  1240. * video: number,
  1241. * id3: number,
  1242. * audioCodec: string,
  1243. * videoCodec: string
  1244. * }}
  1245. *
  1246. * @summary PMT.
  1247. * @property {number} audio
  1248. * Audio PID
  1249. * @property {number} video
  1250. * Video PID
  1251. * @property {number} id3
  1252. * ID3 PID
  1253. * @property {string} audioCodec
  1254. * Audio codec
  1255. * @property {string} videoCodec
  1256. * Video codec
  1257. */
  1258. shaka.util.TsParser.PMT;
  1259. /**
  1260. * @typedef {{
  1261. * channelCount: number,
  1262. * channelConfigCode: number,
  1263. * sampleRate: number
  1264. * }}
  1265. *
  1266. * @summary PMT.
  1267. * @property {number} channelCount
  1268. * @property {number} channelConfigCode
  1269. * @property {number} sampleRate
  1270. */
  1271. shaka.util.TsParser.OpusMetadata;