wip

HLS/MP3Segmenter.php

@@ -0,0 +1,399 @@
+<?php
+/**
+ * .m3u8 playlist generation for HLS (HTTP Live Streaming)
+ *
+ * @file
+ * @ingroup HLS
+ */
+
+namespace MediaWiki\TimedMediaHandler\HLS;
+
+use Exception;
+
+class MP3Segmenter extends Segmenter {
+
+	// http://www.mp3-tech.org/programmer/frame_header.html
+
+	/**
+	 * Internal layout of MP3 frame header bitfield
+	 * @var array
+	 */
+	private static $bits = [
+		'sync'        => [ 21, 11 ],
+		'mpeg'        => [ 19, 2 ],
+		'layer'       => [ 17, 2 ],
+		'protection'  => [ 16, 1 ],
+		'bitrate'     => [ 12, 4 ],
+		'sampleRate'  => [ 10, 2 ],
+		'padding'     => [ 9, 1 ],
+		// below this not needed at present
+		'private'     => [ 8, 1 ],
+		'channelMode' => [ 6, 2 ],
+		'modeExt'     => [ 4, 2 ],
+		'copyright'   => [ 3, 1 ],
+		'original'    => [ 2, 1 ],
+		'emphasis'    => [ 0, 2 ],
+	];
+
+	/**
+	 * 11-bit sync mask for MP3 frame header
+	 * @var int
+	 */
+	private const SYNC_MASK = 0x7ff;
+
+	/**
+	 * Map of sample count per frame based on version/mode
+	 * This is just in case we need to measure non-default sample rates!
+	 * @var array
+	 */
+	private static $samplesPerFrame = [
+		// invalid / layer 3 / 2 / 1
+
+		// MPEG-2.5
+		[ 0, 576, 1152, 384 ],
+		// Reserved
+		[ 0, 0, 0, 0 ],
+		// MPEG-2
+		[ 0, 576, 1152, 384 ],
+		// MPEG-1
+		[ 0, 1152, 384, 384 ],
+	];
+
+	/**
+	 * Map of sample rates based on version/mode
+	 * @var array
+	 */
+	private static $sampleRates = [
+		// MPEG-2.5
+		[ 11025, 12000, 8000, 1 ],
+		// Reserved
+		[ 1, 1, 1, 1 ],
+		// MPEG-2
+		[ 22050, 24000, 16000, 1 ],
+		// MPEG-1
+		[ 44100, 48000, 32000, 1 ],
+	];
+
+	/**
+	 * Map of bit rates based on version/mode/code
+	 * @var array
+	 */
+	private static $bitrates = [
+		// MPEG-2
+		[
+			// invalid layer
+			[ 0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0, ],
+			// layer 3
+			[ 0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0 ],
+			// layer 2
+			[ 0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0 ],
+			// layer 1
+			[ 0, 32, 48, 56, 64, 80, 96, 112, 128, 144, 160, 176, 192, 224, 256, 0 ],
+		],
+		// MPEG-1
+		[
+			// invalid layer
+			[ 0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0, ],
+			// layer 3
+			[ 0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 192, 224, 256, 320, 0 ],
+			// layer 2
+			[ 0, 32, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384, 0 ],
+			// layer 1
+			[ 0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 316, 448, 0 ],
+		]
+	];
+
+	/**
+	 * Timestamp resolution for HLS ID3 timestamp tags
+	 */
+	private const KHZ_90 = 90000;
+
+	/**
+	 * Decode a binary field from the MP3 frame header
+	 */
+	private static function field( string $name, int $header ): int {
+		[ $shift, $bits ] = self::$bits[$name];
+		$mask = ( 1 << $bits ) - 1;
+		return ( $header >> $shift ) & $mask;
+	}
+
+	/**
+	 * Decode an MP3 header bitfield
+	 */
+	private static function frameHeader( string $bytes ): ?array {
+		if ( strlen( $bytes ) < 4 ) {
+			return null;
+		}
+		$data = unpack( "Nval", $bytes );
+		$header = $data['val'];
+
+		// This includes "MPEG 2.5" support, so checks for 11 set bits
+		// not 12 set bits as per original MPEG 1/2
+		$sync = self::field( 'sync', $header );
+		if ( $sync !== self::SYNC_MASK ) {
+			return null;
+		}
+
+		$mpeg = self::field( 'mpeg', $header );
+		$layer = self::field( 'layer', $header );
+		$protection = self::field( 'protection', $header );
+
+		$br = self::field( 'bitrate', $header );
+		$bitrate = 1000 * self::$bitrates[$mpeg & 1][$layer][$br];
+		if ( $bitrate == 0 ) {
+			return null;
+		}
+
+		$sr = self::field( 'sampleRate', $header );
+		$sampleRate = self::$sampleRates[$mpeg][$sr];
+		if ( $sampleRate == 1 ) {
+			return null;
+		}
+
+		$padding = self::field( 'padding', $header );
+
+		$samples = self::$samplesPerFrame[$mpeg][$layer];
+		$duration = $samples / $sampleRate;
+		$nbits = $duration * $bitrate;
+		$nbytes = $nbits / 8;
+		$size = intval( $nbytes );
+
+		if ( $protection == 0 ) {
+			$size += 2;
+		}
+		if ( $padding == 1 ) {
+			$size++;
+		}
+
+		return [
+			'samples' => $samples,
+			'sampleRate' => $sampleRate,
+			'size' => $size,
+			'duration' => $duration,
+		];
+	}
+
+	private static function id3Header( string $bytes ): ?array {
+		// ID3v2.3
+		// https://web.archive.org/web/20081008034714/http://www.id3.org/id3v2.3.0
+		// ID3v2/file identifier   "ID3"
+		// ID3v2 version           $03 00
+		// ID3v2 flags             %abc00000
+		// ID3v2 size              4 * %0xxxxxxx
+		$headerLen = 10;
+		if ( strlen( $bytes ) < $headerLen ) {
+			return null;
+		}
+
+		$data = unpack( "a3tag/nversion/Cflags/C4size", $bytes );
+		if ( $data['tag'] !== 'ID3' ) {
+			return null;
+		}
+
+		$size = $headerLen +
+			( $data['size4'] |
+				( $data['size3'] << 7 ) |
+				( $data['size2'] << 14 ) |
+				( $data['size1'] << 21 ) );
+		return [
+			'size' => $size,
+		];
+	}
+
+	protected function parse(): void {
+		$file = fopen( $this->filename, 'rb' );
+		$stream = new OwningStreamReader( $file );
+
+		$timestamp = 0.0;
+		while ( true ) {
+			$start = $stream->pos();
+			$lookahead = 10;
+			$bytes = $stream->read( $lookahead );
+			if ( $bytes === null ) {
+				// end of file
+				break;
+			}
+
+			// Check for MP3 frame header sync pattern
+			$header = self::frameHeader( $bytes );
+			if ( $header ) {
+				// Note we don't need the data at this time.
+				$stream->seek( $start + $header['size'] );
+				$timestamp += $header['duration'];
+				$this->segments[] = [
+					'start' => $start,
+					'size' => $header['size'],
+					'timestamp' => $timestamp,
+					'duration' => $header['duration'],
+				];
+				continue;
+			}
+
+			// Check for ID3v2 tag
+			$id3 = self::id3Header( $bytes );
+			if ( $id3 ) {
+				// For byte range purposes; count as zero duration
+				$stream->seek( $start + $id3['size'] );
+				$this->segments[] = [
+					'start' => $start,
+					'size' => $id3['size'],
+					'timestamp' => $timestamp,
+					'duration' => 0.0,
+				];
+				continue;
+			}
+
+			throw new Exception( "Not a valid MP3 or ID3 frame at $start" );
+		}
+	}
+
+	/**
+	 * Rewrite the file to include ID3 private tags with timestamp
+	 * data for HLS at segment boundaries. This will modify the file
+	 * in-place and change the segment offsets and sizes in the object.
+	 */
+	public function rewrite(): void {
+		$offset = 0;
+		$id3s = [];
+		$segments = [];
+		foreach ( $this->segments as $i => $orig ) {
+			$id3 = self::timestampTag( $orig['timestamp'] );
+			$delta = strlen( $id3 );
+			$id3s[$i] = $id3;
+			$segments[$i] = [
+				'start' => $orig['start'] + $offset,
+				'size' => $orig['size'] + $delta,
+				'timestamp' => $orig['timestamp'],
+				'duration' => $orig['duration'],
+			];
+			$offset += $delta;
+		}
+
+		$file = fopen( $this->filename, 'rw+b' );
+		$stream = new OwningStreamReader( $file );
+
+		// Move each segment forward, starting at the lastmost to work in-place.
+		$preserveKeys = true;
+		foreach ( array_reverse( $this->segments, $preserveKeys ) as $i => $orig ) {
+			$stream->seek( $orig['start'] );
+			$bytes = $stream->readExactly( $orig['size'] );
+
+			$stream->seek( $segments[$i]['start'] );
+			$stream->write( $id3s[$i] );
+			$stream->write( $bytes );
+		}
+
+		$this->segments = $segments;
+	}
+
+	/**
+	 * Generate an ID3 private tag with a timestamp for use in HLS
+	 * streams of raw media data such as MP3 or AAC.
+	 */
+	protected static function timestampTag( float $timestamp ): string {
+		/*
+		PRIV frame type
+
+		should contain:
+
+		The ID3 PRIV owner identifier MUST be
+		"com.apple.streaming.transportStreamTimestamp".  The ID3 payload MUST
+		be a 33-bit MPEG-2 Program Elementary Stream timestamp expressed as a
+		big-endian eight-octet number, with the upper 31 bits set to zero.
+		Clients SHOULD NOT play Packed Audio Segments without this ID3 tag.
+
+		https://id3.org/id3v2.4.0-frames
+		https://id3.org/id3v2.4.0-structure
+
+		bit order is MSB first, big-endian
+
+		header 10 bytes
+		extended header (var, optional)
+		frames (variable)
+		pading (variable, optional)
+		footer (10 bytes, optional)
+
+
+		header:
+			"ID3"
+			version: 16 bits $04 00
+			flags: 32 bits
+			idv2 size: 32 bits (in chunks of 4 bytes, not counting header or footer)
+
+		flags:
+			bit 7 - unsyncrhonization (??)
+			bit 6 - extended header
+			bit 5 - experimental indicator
+			bit 4 - footer present
+
+		frame:
+			id - 32 bits (four chars)
+			size - 32 bits (in chunks of 4 bytes, excluding frame header)
+			flags - 16 bits
+			(frame data)
+
+		priv payload:
+			owner text string followed by \x00
+			(binary data)
+
+		The timestamps... I think... have 90 kHz integer resolution
+		so convert from the decimal seconds in the HLS
+
+		*/
+
+		$owner = "com.apple.streaming.transportStreamTimestamp\x00";
+		$pts = round( $timestamp * self::KHZ_90 );
+		$thirtyThreeBits = pow( 2, 33 );
+		$thirtyOneBits = pow( 2, 31 );
+		if ( $pts >= $thirtyThreeBits ) {
+			// make sure they won't get too big for 33 bits
+			// this allows about a 24 hour media length
+			throw new Exception( "Timestamp overflow in MP3 output stream: $pts >= $thirtyThreeBits" );
+		}
+		$pts_high = intval( floor( $pts / $thirtyOneBits ) );
+		$pts_low = intval( $pts - ( $pts_high * $thirtyOneBits ) );
+
+		// Private frame payload
+		$frame_data = pack(
+			'a*NN',
+			$owner,
+			$pts_high,
+			$pts_low,
+		);
+
+		// Private frame header
+		$frame_type = 'PRIV';
+		$frame_flags = 0;
+		$frame_length = strlen( $frame_data );
+		if ( $frame_length > 127 ) {
+			throw new Error( "Should never happen: too large ID3 frame data" );
+		}
+		$frame = pack(
+			'a4Nna*',
+			$frame_type,
+			$frame_length,
+			$frame_flags,
+			$frame_data
+		);
+
+		// ID3 tag
+		$tag_type = 'ID3';
+		$tag_version = 0x0400;
+		$tag_flags = 0;
+		// if >127 bytes may need to adjust
+		$tag_length = strlen( $frame );
+		if ( $tag_length > 127 ) {
+			throw new Error( "Should never happen: too large ID3 tag" );
+		}
+		$tag = pack(
+			'a3nCNa*',
+			$tag_type,
+			$tag_version,
+			$tag_flags,
+			$tag_length,
+			$frame
+		);
+
+		return $tag;
+	}
+}

HLS/OwningStreamReader.php

@@ -0,0 +1,21 @@
+<?php
+/**
+ * Base class for streaming media segment readers
+ *
+ * @file
+ * @ingroup HLS
+ */
+
+namespace MediaWiki\TimedMediaHandler\HLS;
+
+/**
+ * Base file class that fcloses on destruct.
+ */
+class OwningStreamReader extends StreamReader {
+	public function __destruct() {
+		if ( $this->file ) {
+			fclose( $this->file );
+			$this->file = null;
+		}
+	}
+}

HLS/Segmenter.php

@@ -0,0 +1,155 @@
+<?php
+/**
+ * Base class for streaming segment readers
+ *
+ * @file
+ * @ingroup HLS
+ */
+
+namespace MediaWiki\TimedMediaHandler\HLS;
+
+use Exception;
+
+/**
+ * Base class for reading a media file and segmenting it.
+ */
+abstract class Segmenter {
+
+	protected string $filename;
+	protected array $segments;
+
+	public function __construct( string $filename, ?array $segments = null ) {
+		$this->filename = $filename;
+		if ( $segments ) {
+			$this->segments = $segments;
+		} else {
+			$this->segments = [];
+			$this->parse();
+		}
+	}
+
+	/**
+	 * Fill the segments from the underlying file
+	 */
+	abstract protected function parse(): void;
+
+	/**
+	 * Consolidate adjacent segments to approach the target segment length.
+	 */
+	public function consolidate( float $target ): void {
+		$out = [];
+		$n = count( $this->segments );
+		$init = $this->segments['init'] ?? false;
+		if ( $init ) {
+			$n--;
+			$out['init'] = $init;
+		}
+		if ( $n < 2 ) {
+			return;
+		}
+
+		$first = $this->segments[0];
+		$start = $first['start'];
+		$size = $first['size'];
+		$timestamp = $first['timestamp'];
+		$duration = $first['duration'];
+
+		$i = 1;
+		while ( $i < $n ) {
+			// Append segments until we get close
+			while ( $i < $n - 1 && $duration < $target ) {
+				$segment = $this->segments[$i];
+				$total = $duration + $segment['duration'];
+				if ( $total >= $target ) {
+					$after = $total - $target;
+					$before = $target - $duration;
+					if ( $before < $after ) {
+						// Break segment early
+						break;
+					}
+				}
+				$duration += $segment['duration'];
+				$size += $segment['size'];
+				$i++;
+			}
+
+			// Save out a segment
+			$out[] = [
+				'start' => $start,
+				'size' => $size,
+				'timestamp' => $timestamp,
+				'duration' => $duration,
+			];
+
+			if ( $i < $n ) {
+				$segment = $this->segments[$i];
+				$start = $segment['start'];
+				$size = $segment['size'];
+				$timestamp = $segment['timestamp'];
+				$duration = $segment['duration'];
+				$i++;
+			}
+		}
+		$out[] = [
+			'start' => $start,
+			'size' => $size,
+			'timestamp' => $timestamp,
+			'duration' => $duration,
+		];
+		$this->segments = $out;
+	}
+
+	/**
+	 * Modify the media file and segments in-place to insert any
+	 * tweaks needed for the file to stream correctly.
+	 *
+	 * This is used by MP3Segmenter to insert ID3 timestamps.
+	 */
+	public function rewrite(): void {
+		// no-op in default; fragmented .mp4 can be left as-is
+	}
+
+	public function playlist( float $target, string $filename ): string {
+		$lines = [];
+		$lines[] = "#EXTM3U";
+		$lines[] = "#EXT-X-VERSION:7";
+		$lines[] = "#EXT-X-TARGETDURATION:$target";
+		$lines[] = "#EXT-MEDIA-SEQUENCE:0";
+		$lines[] = "#EXT-PLAYLIST-TYPE:VOD";
+
+		$url = urlencode( $filename );
+
+		$init = $this->segments['init'] ?? false;
+		if ( $init ) {
+			$lines[] = "#EXT-X-MAP:URI=\"{$url}\",BYTERANGE=\"{$init['size']}@{$init['start']}\"";
+		}
+
+		$n = count( $this->segments ) - 1;
+		for ( $i = 0; $i < $n; $i++ ) {
+			$segment = $this->segments[$i];
+			$lines[] = "#EXTINF:{$segment['duration']},";
+			$lines[] = "#EXT-X-BYTERANGE:{$segment['size']}@{$segment['start']}";
+			$lines[] = "{$url}";
+		}
+
+		$lines[] = "#EXT-X-ENDLIST";
+
+		return implode( "\n", $lines );
+	}
+
+	public static function segment( string $filename ): Segmenter {
+		$ext = strtolower( substr( $filename, strrpos( $filename, '.' ) ) );
+		switch ( $ext ) {
+			case '.mp3':
+				return new MP3Segmenter( $filename );
+			case '.mp4':
+			case '.m4v':
+			case '.m4a':
+			case '.mov':
+			case '.3gp':
+				return new MP4Segmenter( $filename );
+			default:
+				throw new Exception( "Unexpected streaming file extension $ext" );
+		}
+	}
+}

HLS/StreamReader.php

@@ -0,0 +1,104 @@
+<?php
+/**
+ * Base class for streaming media segment readers
+ *
+ * @file
+ * @ingroup HLS
+ */
+
+namespace MediaWiki\TimedMediaHandler\HLS;
+
+use Exception;
+
+/**
+ * Base class for reading/writing a media file with wrappers
+ * for exception handling and possible multi usage.
+ */
+class StreamReader {
+	/**
+	 * @var resource
+	 */
+	protected $file;
+
+	protected int $pos;
+
+	/**
+	 * @param resource $file
+	 */
+	public function __construct( $file ) {
+		if ( get_resource_type( $file ) !== 'stream' ) {
+			throw new Exception( 'Invalid file stream' );
+		}
+		$this->file = $file;
+		$this->pos = $this->tell();
+	}
+
+	private function tell(): int {
+		return ftell( $this->file );
+	}
+
+	public function pos(): int {
+		return $this->pos;
+	}
+
+	/**
+	 * Seek to given absolute file position.
+	 * @throws Exception on error
+	 */
+	public function seek( int $pos ): void {
+		$this->pos = intval( $pos );
+
+		if ( $this->pos === $this->tell() ) {
+			return;
+		}
+		$retval = fseek( $this->file, $this->pos, SEEK_SET );
+
+		if ( $retval < 0 ) {
+			throw new Exception( "Failed to seek to $this->pos bytes" );
+		}
+	}
+
+	/**
+	 * Read $len bytes or return null on EOF/short read.
+	 * @throws Exception on error
+	 */
+	public function read( int $len ): ?string {
+		$this->seek( $this->pos );
+		$bytes = fread( $this->file, $len );
+		if ( $bytes === false ) {
+			throw new Exception( "Read error for $len bytes at $this->pos" );
+		}
+		if ( strlen( $bytes ) < $len ) {
+			return null;
+		}
+		$this->pos += strlen( $bytes );
+		return $bytes;
+	}
+
+	/**
+	 * Read exactly $len bytes
+	 * @throws Exception on error or short read
+	 */
+	public function readExactly( int $len ): string {
+		$bytes = $this->read( $len );
+		if ( $bytes === null ) {
+			throw new Exception( "Short read for $len bytes at $this->pos" );
+		}
+		return $bytes;
+	}
+
+	/**
+	 * Write the given data and return number of bytes written.
+	 * Short writes are possible on network connections, in theory.
+	 * @throws Exception on error
+	 */
+	public function write( string $bytes ): int {
+		$this->seek( $this->pos );
+		$len = strlen( $bytes );
+		$nbytes = fwrite( $this->file, $bytes );
+		if ( $nbytes === false ) {
+			throw new Exception( "Write error for $len bytes at $this->pos" );
+		}
+		return $nbytes;
+	}
+}

HLS/rewrite-mp3.php

@@ -0,0 +1,20 @@
+<?php
+
+require( __DIR__ . '/StreamReader.php' );
+require( __DIR__ . '/OwningStreamReader.php' );
+require( __DIR__ . '/Segmenter.php' );
+require( __DIR__ . '/MP3Segmenter.php' );
+
+use MediaWiki\TimedMediaHandler\HLS\MP3Segmenter;
+
+$argv = $_SERVER['argv'];
+$self = array_shift( $argv );
+$filename = array_shift( $argv );
+$target = 10;
+
+$segmenter = new MP3Segmenter( $filename );
+$segmenter->consolidate( $target );
+$segmenter->rewrite();
+$m3u8 = $segmenter->playlist( $target, $filename );
+
+print $m3u8 . "\n";

fmp4-alt.html

@@ -15,65 +15,62 @@
             <li><a href="fmp4-tracks.html">see component track list</a></li>
         </ul>
 
-        <h3>With fallbacks</h3>
+        <h3>Native HLS</h3>
 
-        <p>HLS with VP9 (.mp4)/MJPEG (.mov) video tracks and Opus/MP3 audio tracks. Custom MSE wrapper enabled to provide HLS-over-MSE for Mac Safari/Chrome/Firefox.</p>
+        <p>HLS with VP9 in mp4, JPEG in mp4, and Opus and AAC in mp4</p>
         <div>
             <video controls width=640 height=360>
-                <source type=application/vnd.apple.mpegurl src=fmp4.vp9-mjpeg.mov.m3u8>
+                <source type=application/vnd.apple.mpegurl src=fmp4.vp9-mjpeg.m3u8>
             </video>
         </div>
 
+        <h3>Overridden HLS</h3>
+
+        <p>HLS with VP8 in mp4, JPEG in mp4, and Opus and AAC in mp4</p>
+        <div>
+            <video class=override controls width=640 height=360>
+                <source type=application/vnd.apple.mpegurl src=fmp4.vp9-mjpeg.m3u8>
+            </video>
+        </div>
+
+
+        <script src="video.js/dist/alt/video.core.js"></script>
+        <script src="http-streaming/dist/videojs-http-streaming.js"></script>
+
         <script type="text/javascript">
+            videojs.log.level('debug');
+            
+            console.log(typeof MediaSource)
             if (typeof MediaSource !== 'undefined') {
-
+                //let vp9 = MediaSource.isTypeSupported('video/mp4; codecs="vp09.00.41.08"');
-                async function process(video) {
+                //let opus = MediaSource.isTypeSupported('video/mp4; codecs="opus"');
-                    let vp9 = 'video/mp4; codecs="vp09.00.41.08"';
+                //let mp3 = MediaSource.isTypeSupported('audio/mpeg');
-                    let opus = 'audio/mp4; codecs="opus"';
+                //if (vp9 && (opus || mp3)) {
-                    let mp3 = 'audio/mpeg';
-
-                    // Temporary hack, just load the full tracks
-                    let videoTracks = [
-                        { type: vp9, src: 'fmp4.480p.vp9.mp4' },
-                    ];
-                    let audioTracks = [
-                        { type: opus, src: 'fmp4.audio.opus.mp4' },
-                        { type: mp3, src: 'fmp4.audio.mpeg.mp3' },
-                    ];
-
-                    let videoTrack = videoTracks.filter(({type}) => MediaSource.isTypeSupported(type))[0];
-                    let audioTrack = audioTracks.filter(({type}) => MediaSource.isTypeSupported(type))[0];
-                    
-                    if (!videoTrack || !audioTrack) {
-                        throw new Error('error no tracks');
-                    }
-                    console.log(videoTrack);
-                    console.log(audioTrack);
-
-                    let videoBytes = await (await fetch(videoTrack.src)).arrayBuffer();
-                    // hackhack
-                    videoBytes = videoBytes.slice(0, 878148 + 779);
-                    let audioBytes = await (await fetch(audioTrack.src)).arrayBuffer();
-
-                    let source = new MediaSource();
-                    source.addEventListener("sourceopen", (event) => {
-                        source.duration = 150; // hack
-                        let videoBuffer = source.addSourceBuffer(videoTrack.type);
-                        let audioBuffer = source.addSourceBuffer(audioTrack.type);
-                        videoBuffer.appendBuffer(videoBytes);
-                        audioBuffer.appendBuffer(audioBytes);
-                        console.log('appended.');
-                    });
-                    video.addEventListener('error', (event) => {
-                        console.log('video error?', video.error);
-                    });
-                    video.src = URL.createObjectURL(source);
-                    console.log('opening...');
-                }
 
                 for (let video of document.querySelectorAll('video')) {
-                    process(video);
+                    let playerConfig = {
+                        responsive: true,
+                        controlBar: {
+                            volumePanel: {
+                                vertical: true,
+                                inline: false
                             }
+                        },
+                        html5: {
+                            vhs: {
+                                // Currently the MP3 audio track fails in Safari
+                                // and it doesn't grok the Opus
+                                // Either fix MP3 handling in vhs or use AAC.
+                                overrideNative: video.classList.contains('override')
+                            }
+                        },
+                    };
+                    video.classList.add('video-js');
+                    video.classList.add('vjs-default-skin');
+                    videojs(video, playerConfig);
+                }
+
+                // }
             }
         </script>
     </body>

fmp4-codecs.html

@@ -10,7 +10,8 @@
 
         <h2>Caminandes - Llamigos</h2>
 
-        <p><a href="fmp4.html">back to main fmp4 entry</a></p>
+        <p><a href="fmp4-tracks.html">see individual tracks</a></p>
+        <p><a href="index.html">back to main fmp4 entry</a></p>
 
         <h3>Single video codecs with Opus .mp4 and MP3 raw audio</h3>
 

fmp4-tracks.html

@@ -10,7 +10,7 @@
 
         <h2>Caminandes - Llamigos</h2>
 
-        <p><a href="fmp4.html">back to main fmp4 entry</a></p>
+        <p><a href="index.html">back to main fmp4 entry</a></p>
         <h2>Component tracks</h2>
 
         <p>VP9 .mp4:</p>
@@ -18,6 +18,11 @@
             <source type=application/vnd.apple.mpegurl src=fmp4.480p.vp9.mp4.m3u8>
         </video>
 
+        <p>VP8 .mp4:</p>
+        <video controls width=640 height=360>
+            <source type=application/vnd.apple.mpegurl src=fmp4.480p.vp8.mp4.m3u8>
+        </video>
+
         <p>MJPEG .mp4:</p>
         <video controls width=640 height=360>
             <source type=application/vnd.apple.mpegurl src=fmp4.120p.mjpeg.mp4.m3u8>

fmp4.html

@@ -1,97 +1 @@
-<!DOCTYPE html>
+<a href="index.html">index.html</a>
-<html>
-    <head>
-        <meta charset=utf-8>
-        <title>HLS VP9/fMP4 test</title>
-        <link rel=stylesheet type=text/css href=video-js/video-js.css>
-    </head>
-    <body>
-        <h1>HLS VP9/fMP4 test</h1>
-
-        <h2>Caminandes - Llamigos</h2>
-
-        <ul>
-            <li><a href="fmp4-codecs.html">see codec variants</a></li>
-            <li><a href="fmp4-tracks.html">see component track list</a></li>
-        </ul>
-
-        <h3>With fallbacks</h3>
-
-        <p>HLS with VP9 (.mp4)/MJPEG (.mov) video tracks and Opus/MP3 audio tracks. Video.js enabled to provide HLS-over-MSE for Chrome/Firefox.</p>
-        <div>
-            <video controls width=640 height=360>
-                <source type=application/vnd.apple.mpegurl src=fmp4.vp9-mjpeg.mov.m3u8>
-            </video>
-        </div>
-
-
-        <p>Current behavior:</p>
-        <p>Browsers that play the VP9 track will get sharp video, those that play the MJPEG track will get blurry video. Audio should sound the same either way.</p>
-        <ul>
-            <li>MSE-based streaming with VHS
-                <ul>
-                    <li>Firefox seems to work with VP9 & Opus tracks via video.js</li>
-                    <li>Chrome works (needed to fix an output setting)</li>
-                    <li>(disabled) Safari uses the MP3 audio tracks and VHS gets confused because it tries to parse them as MP4 (not Safari's fault)</li>
-                </ul>
-            </li>
-            <li>Apple HLS player
-                <ul>
-                    <li>macOS 13's Safari 16 plays MJPEG.</li>
-                    <li>iOS 16 plays VP9 if supported, or MJPEG if no hardware codec</li>
-                    <li>Those last two will also play h.263 or MPEG-4 visual <a href="fmp4-lies.html">IF labeled as if h.264 in the playlist</a>; MJPEG can be properly labeled as "jpeg". I haven't found a supported labeling that is correct yet.</li>
-                    <li><i>no access to iOS 13-15</i></li>
-                    <li>iOS 13 doesn't seem to like mjpeg in .mp4, but .mov is fine</li>
-                    <li>iOS 12 doesn't seem to like any version on an old iPad Air, except with h264 video</li>
-                    <li><i>no access to iOS 11</i></li>
-                    <li>iOS 10 on iPhone 5C plays h.263, or mpeg-4 visual IF labeled as false avc1.blah. It will also play mjpeg if so mislabeled, but only in .mov not in .mp4 as above.</li>
-                    <li>iOS 9 doesn't understand the required version of HLS playlist format, and fails.</li>
-                </ul>
-            </li>
-        </ul>
-
-        <!--<script src="node_modules/video.js/dist/video.js"></script>-->
-
-        <script src="video.js/dist/alt/video.core.js"></script>
-        <script src="http-streaming/dist/videojs-http-streaming.js"></script>
-
-        <script type="text/javascript">
-            let playerConfig = {
-                responsive: true,
-                controlBar: {
-                    volumePanel: {
-                        vertical: true,
-                        inline: false
-                    }
-                },
-                html5: {
-                    vhs: {
-                        // Currently the MP3 audio track fails in Safari
-                        // and it doesn't grok the Opus
-                        // Either fix MP3 handling in vhs or use AAC.
-                        // Switching to fragmented QuickTime ;)
-                        // seems to be helping maybe?
-                        overrideNative: false
-                    }
-                },
-            };
-            videojs.log.level('debug');
-            
-            console.log(typeof MediaSource)
-            if (typeof MediaSource !== 'undefined') {
-                //let vp9 = MediaSource.isTypeSupported('video/mp4; codecs="vp09.00.41.08"');
-                //let opus = MediaSource.isTypeSupported('video/mp4; codecs="opus"');
-                //let mp3 = MediaSource.isTypeSupported('audio/mpeg');
-                //if (vp9 && (opus || mp3)) {
-
-                for (let video of document.querySelectorAll('video')) {
-                    video.classList.add('video-js');
-                    video.classList.add('vjs-default-skin');
-                    videojs(video, playerConfig);
-                }
-
-                // }
-            }
-        </script>
-    </body>
-</html>

index.html

@@ -15,18 +15,29 @@
             <li><a href="fmp4-tracks.html">see component track list</a></li>
         </ul>
 
-        <h3>With fallbacks</h3>
+        <h3>HLS only:</h3>
 
-        <p>HLS with VP9 (.mp4)/MJPEG (.mov) video tracks and Opus/MP3 audio tracks. Video.js enabled to provide HLS-over-MSE for Chrome/Firefox.</p>
+        <p>HLS with correctly-labeled VP9 (.mp4) and MJPEG (.mov) video tracks and Opus (.mp4, for Chrome/Firefox) and MP3 (.mp3, used by Safari) audio tracks. Video.js is enabled but will not yet override the native HLS in Safari Desktop.</p>
         <div>
             <video controls width=640 height=360>
                 <source type=application/vnd.apple.mpegurl src=fmp4.vp9-mjpeg.mov.m3u8>
             </video>
         </div>
 
+        <h3>WebM or HLS:</h3>
+        <p>WebM VP9/Opus, WebM VP8/Vorbis, or HLS. Current versions of desktop Safari should see one or the other WebM, and those that fall back to the HLS may or may not work.</p>
+        <div>
+            <video controls width=640 height=360>
+                <source type="video/webm; codecs=&quot;vp9, opus&quot;" src=flat.480p.vp9-opus.webm>
+                <source type="video/webm; codecs=&quot;vp8, vorbis&quot;" src=flat.480p.vp8-vorbis.webm>
+                <source type=application/vnd.apple.mpegurl src=fmp4.vp9-mjpeg.mov.m3u8>
+            </video>
+        </div>
+
+
 
         <p>Current behavior:</p>
-        <p>Browsers that play the VP9 track will get sharp video, those that play the MJPEG track will get blurry video. Audio should sound the same either way.</p>
+        <p>Browsers that play the VP9 track or one of the WebM files will get sharp video, those that play the MJPEG track will get blurry video. Audio should sound the same either way.</p>
         <ul>
             <li>MSE-based streaming with VHS
                 <ul>
@@ -36,17 +47,78 @@
                 </ul>
             </li>
             <li>Apple HLS player
-                <ul>
+                <table border>
-                    <li>macOS 13's Safari 16 plays MJPEG.</li>
+                    <tr>
-                    <li>iOS 16 plays VP9 if supported, or MJPEG if no hardware codec</li>
+                        <th>platform</th>
-                    <li>Those last two will also play h.263 or MPEG-4 visual <a href="fmp4-lies.html">IF labeled as if h.264 in the playlist</a>; MJPEG can be properly labeled as "jpeg". I haven't found a supported labeling that is correct yet.</li>
+                        <td>vp09 in .mp4</td>
-                    <li><i>no access to iOS 13-15</i></li>
+                        <td>mp4v in .mp4</td>
-                    <li>iOS 13 doesn't seem to like mjpeg in .mp4, but .mov is fine</li>
+                        <td>s263 in .3gp</td>
-                    <li>iOS 12 doesn't seem to like any version on an old iPad Air, except with h264 video</li>
+                        <td>jpeg in .mov</td>
-                    <li><i>no access to iOS 11</i></li>
+                        <td>jpeg in .mp4</td>
-                    <li>iOS 10 on iPhone 5C plays h.263, or mpeg-4 visual IF labeled as false avc1.blah. It will also play mjpeg if so mislabeled, but only in .mov not in .mp4 as above.</li>
+                    </tr>
-                    <li>iOS 9 doesn't understand the required version of HLS playlist format, and fails.</li>
+                    <tr>
-                </ul>
+                        <td>
+                            macOS 13 / Safari 16.4<br>
+                            on 2019 Intel MBP, 2020 M1 MBA
+                        </td>
+                        <td></td>
+                        <td>if lie</td>
+                        <td>if lie</td>
+                        <td>works</td>
+                        <td>works in 16.4 only?</td>
+                    </tr>
+                    <tr>
+                        <td>
+                            macOS 12 / Safari 16.4<br>
+                            on 2015 Intel MBP, 2015 Intel MBA
+                        </td>
+                        <td></td>
+                        <td>if lie</td>
+                        <td>if lie</td>
+                        <td>if lie</td>
+                        <td></td>
+                    </tr>
+                    <tr>
+                        <td>
+                            iOS 13+ on A12+
+                        </td>
+                        <td>works</td>
+                        <td>works</td>
+                        <td>if lie</td>
+                        <td>if lie</td>
+                        <td>works in 16.4 only?</td>
+                    </tr>
+                    <tr>
+                        <td>
+                            iOS 13+ on A11+
+                        </td>
+                        <td></td>
+                        <td>works</td>
+                        <td>if lie</td>
+                        <td>if lie</td>
+                        <td>works in 16.4 only?</td>
+                    </tr>
+                    <tr>
+                        <td>
+                            iOS 12 on A7
+                        </td>
+                        <td></td>
+                        <td></td>
+                        <td></td>
+                        <td></td>
+                        <td></td>
+                    </tr>
+                    <tr>
+                        <td>
+                            iOS 10 on A6
+                        </td>
+                        <td></td>
+                        <td>if lie</td>
+                        <td>if lie</td>
+                        <td>if lie</td>
+                        <td></td>
+                    </tr>
+                </table>
             </li>
         </ul>
 

make-fmp4.sh

@@ -13,8 +13,10 @@ VIDEO_MPEG4="-vcodec mpeg4 -g 240 $BITRATE_HI $SIZE_MAIN"
 VIDEO_H263="-vcodec h263 -g 240 $BITRATE_LO $SIZE_H263"
 VIDEO_MJPEG="-vcodec mjpeg $BITRATE_HI $SIZE_SMALL"
 VIDEO_VP9="-vcodec libvpx-vp9 -tile-columns 2 -row-mt 1 -cpu-used 3 -g 240 $BITRATE_LO $SIZE_MAIN"
+VIDEO_VP8="-vcodec libvpx -slices 4 -cpu-used 3 -g 240 $BITRATE_HI $SIZE_MAIN"
 
 AUDIO_OPUS="-acodec libopus -ac 2 -ar 48000 -ab 96k"
+AUDIO_VORBIS="-acodec libvorbis -ac 2 -ar 48000 -ab 128k"
 AUDIO_AAC="-ac 2 -ar 48000 -ab 128k"
 AUDIO_MP3="-acodec libmp3lame -ac 2 -ar 48000 -ab 128k"
 
@@ -23,12 +25,13 @@ INFILE=caminandes-llamigos.webm
 set -e
 
 # Audio for HLS
-#ffmpeg -i $INFILE -vn $AUDIO_MP3 -y fmp4.audio.mpeg.mp3
+# note - must make the MP3 because we have to reprocess it!
+ffmpeg -i $INFILE -vn $AUDIO_MP3 -y fmp4.audio.mpeg.mp3
+
 #ffmpeg -i $INFILE -vn $AUDIO_MP3 $AUDFLAGS -y fmp4.audio.mpeg.mp4
 #ffmpeg -i $INFILE -vn $AUDIO_MP3 $AUDFLAGS -y fmp4.audio.mpeg.mov
 #ffmpeg -i $INFILE -vn $AUDIO_AAC $AUDFLAGS  -y fmp4.audio.aac.mp4
 #ffmpeg -i $INFILE -vn $AUDIO_OPUS $AUDFLAGS -y fmp4.audio.opus.mp4
-ffmpeg -i $INFILE -vn $AUDIO_OPUS $AUDFLAGS -y fmp4.audio.alac.mp4
 
 # Video for HLS
 
@@ -48,9 +51,15 @@ ffmpeg -i $INFILE -vn $AUDIO_OPUS $AUDFLAGS -y fmp4.audio.alac.mp4
 #ffmpeg -i $INFILE -an $VIDEO_MJPEG $AUDFLAGS -y fmp4.120p.mjpeg.mp4
 #ffmpeg -i $INFILE -an $VIDEO_MJPEG $AUDFLAGS -y fmp4.120p.mjpeg.mov
 
+# Flat fallbacks
+#ffmpeg -i $INFILE $AUDIO_OPUS $VIDEO_VP9 -pass 1 -y flat.480p.vp9-opus.webm
+#ffmpeg -i $INFILE $AUDIO_OPUS $VIDEO_VP9 -pass 2 -y flat.480p.vp9-opus.webm
+#ffmpeg -i $INFILE $AUDIO_VORBIS $VIDEO_VP8 -pass 1 -y flat.480p.vp8-vorbis.webm
+#ffmpeg -i $INFILE $AUDIO_VORBIS $VIDEO_VP8 -pass 2 -y flat.480p.vp8-vorbis.webm
+
 
 # Playlist processing
-php extract-playlist.php fmp4.audio.mpeg.mp3 > fmp4.audio.mpeg.mp3.m3u8
+php HLS/rewrite-mp3.php fmp4.audio.mpeg.mp3 > fmp4.audio.mpeg.mp3.m3u8
 php extract-playlist.php fmp4.audio.mpeg.mp4 > fmp4.audio.mpeg.mp4.m3u8
 php extract-playlist.php fmp4.audio.mpeg.mov > fmp4.audio.mpeg.mov.m3u8
 php extract-playlist.php fmp4.audio.aac.mp4 > fmp4.audio.aac.mp4.m3u8

meta-playlist.php

@@ -26,6 +26,7 @@ $audioCodecs = [
 // @fixme use correct settings based on the file
 $videoCodecs = [
     'vp9'  => 'vp09.00.41.08',
+    'vp8'  => 'vp08.0.0.08',
     'h264' => 'avc1.42e00a',
 
     // truths
@@ -41,6 +42,7 @@ if ( $lie ) {
     $videoCodecs['mjpeg'] = $lie;
     $videoCodecs['h263' ] = $lie;
     $videoCodecs['mpeg4'] = $lie;
+    $videoCodecs['vp8'] = $lie;
 }
 
 

test-rewrite.sh

@@ -0,0 +1,2 @@
+cp fmp4.audio.mpeg.mp3 rewrite.audio.mpeg.mp3
+php HLS/rewrite-mp3.php rewrite.audio.mpeg.mp3