Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

通过websocket一帧一帧的传数据,我该如何封装 #57

Open
zileyuan opened this issue Mar 15, 2023 · 29 comments
Open

通过websocket一帧一帧的传数据,我该如何封装 #57

zileyuan opened this issue Mar 15, 2023 · 29 comments

Comments

@zileyuan
Copy link

zileyuan commented Mar 15, 2023

情况是这样的,服务端把直播流一帧一帧的推过来,我也需要通过gomedia封装成FMp4,然后封装成wasm,让web页面得到封装好的fmp4数据包,我用了下面两种方法

type WsFrame struct {
	codec     mp4.MP4_CODEC_TYPE
	timestamp uint64
	source    []byte  //加了一个16位的头
	raw       []byte  // 真正的帧数据
}
type Muxer struct {
	cache []byte
}
func (m *Muxer) bothMuxer1(muxer *mp4.Movmuxer, frame *WsFrame) {
	var tid uint32
	if frame.codec < mp4.MP4_CODEC_AAC {
		tid = muxer.AddVideoTrack(frame.codec)
		muxer.Write(tid, frame.raw, frame.timestamp, frame.timestamp)
		muxer.WriteTrailer()
		fmt.Printf("frame.timestamp %v\n", frame.timestamp)
	} else {
		//tid = muxer.AddAudioTrack(frame.codec, mp4.WithAudioSampleRate(frame.sampleRate), mp4.WithAudioChannelCount(frame.channelCount))
	}
}

func (m *Muxer) bothMuxer2(muxer *mp4.Movmuxer, frame *WsFrame) {
	var tid uint32
	if frame.codec < mp4.MP4_CODEC_AAC {
		tid = muxer.AddVideoTrack(frame.codec)
		var isVCLNaluType bool
		if frame.codec == mp4.MP4_CODEC_H264 {
			ntype := codec.H264NaluType(frame.raw)
			isVCLNaluType = codec.IsH264VCLNaluType(ntype)
		} else if frame.codec == mp4.MP4_CODEC_H265 {
			ntype := codec.H265NaluType(frame.raw)
			isVCLNaluType = codec.IsH265VCLNaluType(ntype)
		}
		if !isVCLNaluType {
			m.cache = append(m.cache, frame.raw...)
			return
		}
		if len(m.cache) > 0 {
			m.cache = append(m.cache, frame.raw...)
			muxer.Write(tid, m.cache, frame.timestamp, frame.timestamp)
			m.cache = m.cache[:0]
		} else {
			muxer.Write(tid, frame.raw, frame.timestamp, frame.timestamp)
		}
		muxer.WriteTrailer()
		fmt.Printf("frame.timestamp %v\n", frame.timestamp)
	} else {
		// tid = muxer.AddAudioTrack(frame.codec, mp4.WithAudioSampleRate(frame.sampleRate), mp4.WithAudioChannelCount(frame.channelCount))
		// muxer.Write(tid, frame.raw, frame.timestamp, frame.timestamp)
		// muxer.WriteTrailer()
	}
}

我暂时把bothMuxer1和bothMuxer2 中的音频部分去掉了,但是产生的fmap4数据包,我放到web的video表上播放,会出现解析错误(CHUNK_DEMUXER_ERROR_APPEND_FAILED: RunSegmentParserLoop: stream parsing failed. append_window_start=0 append_window_end=inf),但是之前我用多个帧生成的fmp4数据放在video就是可以正常播放,请问是什么原因,是不是我封装的方式还是有问题?

@zileyuan zileyuan changed the title 通过websocket一帧一帧的把数据,我该如何封装 通过websocket一帧一帧的传数据,我该如何封装 Mar 15, 2023
@yapingcat
Copy link
Owner

yapingcat commented Mar 15, 2023

我看你的场景是流式的,可以参考下面的伪代码

var firstSegment = true
var firstFrame = true
var  vtid = 0
func (m *Muxer) bothMuxer2(muxer *mp4.Movmuxer, frame *WsFrame) {

	 // 以MP4_FLAG_DASH  创建muxer
	//muxer,_ := mp4.CreateMp4Muxer(mp4file, mp4.WithMp4Flag(mp4.MP4_FLAG_DASH))
	muxer.OnNewFragment(func(duration uint32, firstPts, firstDts uint64) {
		if firstSegment {
                       //第一个fmp4切片分段
			b := make([]byte, 4096)
			initSeg := bytes.NewBuffer(b)
			muxer.WriteInitSegment(initSeg)
			firstSegment = false
			// 把initSeg.Bytes() 喂给浏览器
		}
	})

	if frame.codec == mp4.MP4_CODEC_H264 {
                if firstFrame {
                       vtid = muxer.AddVideoTrack(mp4.MP4_CODEC_H264)
                       firstFrame = false
                 }
		ntype := codec.H264NaluType(frame.raw)
		isVCLNaluType = codec.IsH264VCLNaluType(ntype)
	} else if frame.codec == mp4.MP4_CODEC_H265 {
                if firstFrame {
                       vtid = muxer.AddVideoTrack(mp4.MP4_CODEC_H265)
                       firstFrame = false
                 }
		ntype := codec.H265NaluType(frame.raw)
		isVCLNaluType = codec.IsH265VCLNaluType(ntype)
	}
	if !isVCLNaluType {
		m.cache = append(m.cache, frame.raw...)
		return
	}
	if len(m.cache) > 0 {
		m.cache = append(m.cache, frame.raw...)
		muxer.Write(vtid, m.cache, frame.timestamp, frame.timestamp)
		m.cache = m.cache[:0]
	} else {
		muxer.Write(vtid, frame.raw, frame.timestamp, frame.timestamp)
	}
       //每写入一帧vcl数据,调用一次FlushFragment,相应的会生成一个fmp4切片
	muxer.FlushFragment()
        ........ //这一步把mp4file中的fmp4数据喂给浏览器  
        
        //创建新的mp4file 
        newmp4file := NewMp4ioxxxx()
	muxer.ReBindWriter(newmp4file)
       //mp4file可以是文件io,也可以是内存io,取决你的实现,内存io可以参考example_mux_mp4_memory_io.go 实现
}

@zileyuan
Copy link
Author

zileyuan commented Mar 16, 2023

首先还是感谢,我完整的贴一下,我的代码,我按上面代码调整了一下,没有进入OnNewFragment的回调,现在ws每次只发一帧过来,我原来的做法是 直接把 fws.buffer 喂给前端video

type fmp4WriterSeeker struct {
	buffer []byte
	offset int
}

func newFmp4WriterSeeker(capacity int) *fmp4WriterSeeker {
	return &fmp4WriterSeeker{
		buffer: make([]byte, 0, capacity),
		offset: 0,
	}
}

func (fws *fmp4WriterSeeker) Write(p []byte) (n int, err error) {
	if cap(fws.buffer)-fws.offset >= len(p) {
		if len(fws.buffer) < fws.offset+len(p) {
			fws.buffer = fws.buffer[:fws.offset+len(p)]
		}
		copy(fws.buffer[fws.offset:], p)
		fws.offset += len(p)
		return len(p), nil
	}
	tmp := make([]byte, len(fws.buffer), cap(fws.buffer)+len(p)*2)
	copy(tmp, fws.buffer)
	if len(fws.buffer) < fws.offset+len(p) {
		tmp = tmp[:fws.offset+len(p)]
	}
	copy(tmp[fws.offset:], p)
	fws.buffer = tmp
	fws.offset += len(p)
	return len(p), nil
}

func (fws *fmp4WriterSeeker) Seek(offset int64, whence int) (int64, error) {
	if whence == io.SeekCurrent {
		if fws.offset+int(offset) > len(fws.buffer) {
			return -1, errors.New(fmt.Sprint("SeekCurrent out of range", len(fws.buffer), offset, fws.offset))
		}
		fws.offset += int(offset)
		return int64(fws.offset), nil
	} else if whence == io.SeekStart {
		if offset > int64(len(fws.buffer)) {
			return -1, errors.New(fmt.Sprint("SeekStart out of range", len(fws.buffer), offset, fws.offset))
		}
		fws.offset = int(offset)
		return offset, nil
	} else {
		return 0, errors.New("unsupport SeekEnd")
	}
}
type WsFrame struct {
	codec     mp4.MP4_CODEC_TYPE
	timestamp uint64
	source    []byte  //加了一个16位的头
	raw       []byte  // 真正的帧数据
}
func (frame *WsFrame) parse() {
	codec := frame.source[0:4]
	switch {
	case bytes.Equal(codec, []byte{0x00, 0x00, 0x00, 0x01}): // 265
		frame.codec = mp4.MP4_CODEC_H265
	case bytes.Equal(codec, []byte{0x00, 0x00, 0x00, 0x00}): // 264
		frame.codec = mp4.MP4_CODEC_H264
	case bytes.Equal(codec, []byte{0x00, 0x00, 0x00, 0x25}): // 37
		frame.codec = mp4.MP4_CODEC_AAC
	case bytes.Equal(codec, []byte{0x00, 0x00, 0x00, 0x13}): // 19
		frame.codec = mp4.MP4_CODEC_G711A
	case bytes.Equal(codec, []byte{0x13, 0x00, 0x00, 0x14}): // 20
		frame.codec = mp4.MP4_CODEC_G711U
	}
	frame.timestamp = binary.BigEndian.Uint64(frame.source[4:12])
	frame.raw = frame.source[16:]
	fmt.Printf("codec %v\n", codec)
	fmt.Printf("frame.codec %v\n", frame.codec)
	fmt.Printf("frame.timestamp %v\n", frame.timestamp)
}
type Muxer struct {
	cache []byte
}
func createFMp4Muxer() (*mp4.Movmuxer, *fmp4WriterSeeker) {
	fws := newFmp4WriterSeeker(1024 * 1024)
	muxer, err := mp4.CreateMp4Muxer(fws, mp4.WithMp4Flag(mp4.MP4_FLAG_FRAGMENT))
	if err != nil {
		fmt.Println(err)
		return nil, nil
	}
	return muxer, fws
}
func (m *Muxer) doBothMuxer(frame *WsFrame) ([]byte, error) {
	muxer, fws := createFMp4Muxer()
	err := m.bothMuxer(muxer, frame)
	return fws.buffer, err
}
func (m *Muxer) bothMuxer(muxer *mp4.Movmuxer, frame *WsFrame) error {
	var tid uint32
	if frame.codec < mp4.MP4_CODEC_AAC {
		muxer.OnNewFragment(func(duration uint32, firstPts, firstDts uint64) {
			fmt.Printf("----------OnNewFragment\n")
			//第一个fmp4切片分段
			b := make([]byte, 4096)
			initSeg := bytes.NewBuffer(b)
			muxer.WriteInitSegment(initSeg)
			// 把initSeg.Bytes() 喂给浏览器

		})
		tid = muxer.AddVideoTrack(frame.codec)
		var isVCLNaluType bool
		if frame.codec == mp4.MP4_CODEC_H264 {
			ntype := codec.H264NaluType(frame.raw)
			isVCLNaluType = codec.IsH264VCLNaluType(ntype)
		} else if frame.codec == mp4.MP4_CODEC_H265 {
			ntype := codec.H265NaluType(frame.raw)
			isVCLNaluType = codec.IsH265VCLNaluType(ntype)
		}
		if !isVCLNaluType {
			m.cache = append(m.cache, frame.raw...)
			fmt.Printf("----------!isVCLNaluType\n")
			return errors.New("67899")
		}
		if len(m.cache) > 0 {
			m.cache = append(m.cache, frame.raw...)
			muxer.Write(tid, m.cache, frame.timestamp, frame.timestamp)
			m.cache = m.cache[:0]
		} else {
			muxer.Write(tid, frame.raw, frame.timestamp, frame.timestamp)
		}
		muxer.FlushFragment()
	} else {
		return errors.New("12345")
		// tid = muxer.AddAudioTrack(frame.codec, mp4.WithAudioSampleRate(frame.sampleRate), mp4.WithAudioChannelCount(frame.channelCount))
		// if frame.codec == mp4.MP4_CODEC_AAC {
		// 	interval := uint64((1024 * 1000 / frame.sampleRate))
		// 	timestamp := frame.timestamp
		// 	codec.SplitAACFrame(frame.raw, func(bytes []byte) {
		// 		muxer.Write(tid, bytes, timestamp, timestamp)
		// 		timestamp += interval
		// 	})
		// } else if frame.codec == mp4.MP4_CODEC_G711A || frame.codec == mp4.MP4_CODEC_G711U {
		// 	muxer.Write(tid, frame.raw, frame.timestamp, frame.timestamp)
		// }
		// muxer.WriteTrailer()
	}
	return nil
}
func (m *Muxer) process(data js.Value) interface{} {
	fmt.Printf("data %v\n", data)
	result := make(map[string]interface{})

	byteMessage := make([]byte, data.Length())
	js.CopyBytesToGo(byteMessage, data)
	fmt.Printf("byteMessage %v\n", byteMessage)
	frame := &WsFrame{
		source: byteMessage,
	}
	frame.parse()
	fmt.Printf("rawData %v\n", frame.raw)
	muxterData, err := m.doBothMuxer(frame)
	if err == nil {
		fmt.Printf("muxterData %v\n", muxterData)
		streamResult := js.Global().Get("Uint8Array").New(len(muxterData))
		js.CopyBytesToJS(streamResult, muxterData)
		fmt.Printf("streamResult %v\n", streamResult)
		result["stream"] = streamResult
	}
	return result
}

@zileyuan
Copy link
Author

是不是可以这样理解,我需要把muxer存放在内存中,不停的接受定时的Wsframe,然后每来一帧vcl数据,调用一次FlushFragment,然后他会进入OnNewFragment,我获取initSeg返回给浏览器,我有一个疑问,我创建的fmp4WriterSeeker对象是不是在FlushFragment后 是不是下次就要重新创建?

@zileyuan
Copy link
Author

我调用muxer.FlushFragment()不会进入OnNewFragment回调,调用muxer.WriteTrailer()会进入回调,不过也就进一次

@zileyuan
Copy link
Author

使用 mp4.MP4_FLAG_DASH 创建muxer,然后使用muxer.FlushFragment()不会进入OnNewFragment 回调

@yapingcat
Copy link
Owner

yapingcat commented Mar 16, 2023

使用 mp4.MP4_FLAG_DASH 创建muxer,然后使用muxer.FlushFragment()不会进入OnNewFragment 回调

非常抱歉, 是的,OnNewFragment 一开始是针对dash和hls开发的,你这个场景还不太一样,
我写了个demo,读取的是flv数据源,然后按照你的场景一帧一个fmp4切片,麻烦参考一下试试,有问题及时交流

package main

import (
	"errors"
	"fmt"
	"io"
	"os"

	"github.com/yapingcat/gomedia/go-codec"
	"github.com/yapingcat/gomedia/go-flv"
	"github.com/yapingcat/gomedia/go-mp4"
)

type fmp4WriterSeeker struct {
	buffer []byte
	offset int
}

func newFmp4WriterSeeker(capacity int) *fmp4WriterSeeker {
	return &fmp4WriterSeeker{
		buffer: make([]byte, 0, capacity),
		offset: 0,
	}
}

func (fws *fmp4WriterSeeker) Write(p []byte) (n int, err error) {
	if cap(fws.buffer)-fws.offset >= len(p) {
		if len(fws.buffer) < fws.offset+len(p) {
			fws.buffer = fws.buffer[:fws.offset+len(p)]
		}
		copy(fws.buffer[fws.offset:], p)
		fws.offset += len(p)
		return len(p), nil
	}
	tmp := make([]byte, len(fws.buffer), cap(fws.buffer)+len(p)*2)
	copy(tmp, fws.buffer)
	if len(fws.buffer) < fws.offset+len(p) {
		tmp = tmp[:fws.offset+len(p)]
	}
	copy(tmp[fws.offset:], p)
	fws.buffer = tmp
	fws.offset += len(p)
	return len(p), nil
}

func (fws *fmp4WriterSeeker) Seek(offset int64, whence int) (int64, error) {
	if whence == io.SeekCurrent {
		if fws.offset+int(offset) > len(fws.buffer) {
			return -1, errors.New(fmt.Sprint("SeekCurrent out of range", len(fws.buffer), offset, fws.offset))
		}
		fws.offset += int(offset)
		return int64(fws.offset), nil
	} else if whence == io.SeekStart {
		if offset > int64(len(fws.buffer)) {
			return -1, errors.New(fmt.Sprint("SeekStart out of range", len(fws.buffer), offset, fws.offset))
		}
		fws.offset = int(offset)
		return offset, nil
	} else {
		return 0, errors.New("unsupport SeekEnd")
	}
}

type Frame struct {
	codec mp4.MP4_CODEC_TYPE
	frame []byte
	pts   uint64
	dts   uint64
}

type Muxer struct {
	cache []byte
	first bool
	i     int
	muxer *mp4.Movmuxer
	fws   *fmp4WriterSeeker
	tid   uint32
}

func NewMuxer() *Muxer {
	m := &Muxer{
		first: true,
	}
	m.fws = newFmp4WriterSeeker(1024 * 1024)
	m.muxer, _ = mp4.CreateMp4Muxer(m.fws, mp4.WithMp4Flag(mp4.MP4_FLAG_DASH))
	return m
}

func (m *Muxer) doBothMuxer(frame *Frame) ([]byte, error) {
	err := m.bothMuxer(frame)
	filename := fmt.Sprintf("stream-%d.mp4", m.i)
	mp4file, _ := os.OpenFile(filename, os.O_CREATE|os.O_RDWR, 0666)
	mp4file.Write(m.fws.buffer)
	b := m.fws.buffer
	m.fws = newFmp4WriterSeeker(1024 * 1024)
	m.muxer.ReBindWriter(m.fws)
	return b, err
}
func (m *Muxer) bothMuxer(frame *Frame) error {
	if frame.codec < mp4.MP4_CODEC_AAC {
		if m.first {
			m.tid = m.muxer.AddVideoTrack(frame.codec)
		}
		var isVCLNaluType bool
		if frame.codec == mp4.MP4_CODEC_H264 {
			ntype := codec.H264NaluType(frame.frame)
			isVCLNaluType = codec.IsH264VCLNaluType(ntype)
		} else if frame.codec == mp4.MP4_CODEC_H265 {
			ntype := codec.H265NaluType(frame.frame)
			isVCLNaluType = codec.IsH265VCLNaluType(ntype)
		}
		if !isVCLNaluType {
			m.cache = append(m.cache, frame.frame...)
			fmt.Printf("----------!isVCLNaluType\n")
			return errors.New("67899")
		}
		if len(m.cache) > 0 {
			m.cache = append(m.cache, frame.frame...)
			m.muxer.Write(m.tid, m.cache, frame.pts, frame.dts)
			m.cache = m.cache[:0]
		} else {
			m.muxer.Write(m.tid, frame.frame, frame.pts, frame.dts)
		}
		if m.first {
			m.first = false
			initFile, _ := os.OpenFile("init.mp4", os.O_CREATE|os.O_RDWR, 0666)
			m.muxer.WriteInitSegment(initFile)
		}
		m.muxer.FlushFragment()

	} else {
		return errors.New("12345")
		// tid = muxer.AddAudioTrack(frame.codec, mp4.WithAudioSampleRate(frame.sampleRate), mp4.WithAudioChannelCount(frame.channelCount))
		// if frame.codec == mp4.MP4_CODEC_AAC {
		// 	interval := uint64((1024 * 1000 / frame.sampleRate))
		// 	timestamp := frame.timestamp
		// 	codec.SplitAACFrame(frame.raw, func(bytes []byte) {
		// 		muxer.Write(tid, bytes, timestamp, timestamp)
		// 		timestamp += interval
		// 	})
		// } else if frame.codec == mp4.MP4_CODEC_G711A || frame.codec == mp4.MP4_CODEC_G711U {
		// 	muxer.Write(tid, frame.raw, frame.timestamp, frame.timestamp)
		// }
		// muxer.WriteTrailer()
	}
	return nil
}

func main() {

	flvfilereader, _ := os.Open(os.Args[1])
	defer flvfilereader.Close()
	fr := flv.CreateFlvReader()

	m := NewMuxer()
	i := 0
	fr.OnFrame = func(ci codec.CodecID, b []byte, pts, dts uint32) {
		if ci == codec.CODECID_VIDEO_H264 {
			f := &Frame{
				codec: mp4.MP4_CODEC_H264,
				pts:   uint64(pts),
				dts:   uint64(dts),
				frame: b,
			}
			r, _ := m.doBothMuxer(f)
			filename := fmt.Sprintf("stream-%d.mp4", i)
			mp4file, _ := os.OpenFile(filename, os.O_CREATE|os.O_RDWR, 0666)
			mp4file.Write(r)
			mp4file.Close()
			i++
		}
	}

	cache := make([]byte, 4096)
	for {
		n, err := flvfilereader.Read(cache)
		if err != nil {
			fmt.Println(err)
			break
		}
		fr.Input(cache[0:n])
	}

}

@zileyuan
Copy link
Author

zileyuan commented Mar 16, 2023

非常感谢,我看了一下代码,是不是,第一帧的数据是init.mp4里面的 []byte,后续的数据是stream-%d.mp4的[]byte,我都按顺序给到web页面的mse中,是吧,我试试

@yapingcat
Copy link
Owner

非常感谢,我看了一下代码,是不是,第一帧的数据是init.mp4里面的 []byte,后续的数据是stream-%d.mp4的[]byte,我都按顺序给到web页面的mse中,是吧,我试试

是的

@zileyuan
Copy link
Author

[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 28 102 116 121 112 105 115 111 53 0 0 2 0 105 115 111 53 105 115 111 54 109 112 52 49 0 0 2 139 109 111 111 118 0 0 0 108 109 118 104 100 0 0 0 0 224 57 15 215 224 57 15 215 0 0 3 232 0 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 64 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 1 239 116 114 97 107 0 0 0 92 116 107 104 100 0 0 0 3 224 57 15 215 224 57 15 215 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 64 0 0 0 2 192 0 0 2 64 0 0 0 0 1 139 109 100 105 97 0 0 0 32 109 100 104 100 0 0 0 0 224 57 15 215 224 57 15 215 0 0 3 232 0 0 0 0 85 196 0 0 0 0 0 45 104 100 108 114 0 0 0 0 0 0 0 0 118 105 100 101 0 0 0 0 0 0 0 0 0 0 0 0 86 105 100 101 111 72 97 110 100 108 101 114 0 0 0 1 54 109 105 110 102 0 0 0 20 118 109 104 100 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 36 100 105 110 102 0 0 0 28 100 114 101 102 0 0 0 0 0 0 0 1 0 0 0 12 117 114 108 32 0 0 0 1 0 0 0 246 115 116 98 108 0 0 0 150 115 116 115 100 0 0 0 0 0 0 0 1 0 0 0 134 97 118 99 49 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 192 2 64 0 72 0 0 0 72 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 24 255 255 0 0 0 48 97 118 99 67 1 77 96 52 255 225 0 25 39 77 96 52 141 141 80 88 9 52 217 0 0 3 0 1 0 0 3 0 50 15 22 38 160 1 0 4 40 238 9 200 0 0 0 16 115 116 116 115 0 0 0 0 0 0 0 0 0 0 0 16 115 116 115 99 0 0 0 0 0 0 0 0 0 0 0 20 115 116 115 122 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 16 115 116 99 111 0 0 0 0 0 0 0 0 0 0 0 20 115 116 115 115 0 0 0 0 0 0 0 1 0 0 0 1 0 0 0 40 109 118 101 120 0 0 0 32 116 114 101 120 0 0 0 0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]
我获取了一下 init.mp4 的数据,是上面,感觉好像不太对,我塞到mse 里面也出错了MEDIA_ERR_SRC_NOT_SUPPORTED

@zileyuan
Copy link
Author

[0 0 0 24 115 116 121 112 109 115 100 104 0 0 0 0 109 115 100 104 109 115 105 120 0 0 0 92 115 105 100 120 1 0 0 0 0 0 0 1 0 0 3 232 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 52 0 0 0 1 0 0 0 230 233 248 71 159 128 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 108 109 111 111 102 0 0 0 16 109 102 104 100 0 0 0 0 0 0 0 3 0 0 0 84 116 114 97 102 0 0 0 32 116 102 104 100 0 2 0 58 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 114 1 1 0 0 0 0 0 20 116 102 100 116 1 0 0 0 0 0 1 134 233 248 71 159 0 0 0 24 116 114 117 110 1 0 1 1 0 0 0 1 0 0 0 116 0 0 0 0 0 0 0 122 109 100 97 116 0 0 0 110 33 224 0 96 0 110 74 224 139 255 36 127 69 169 37 60 3 75 238 171 231 171 133 66 186 228 169 241 55 11 18 140 197 142 176 252 47 131 58 154 243 94 129 137 245 91 234 190 194 65 158 6 24 125 131 42 152 22 227 33 147 231 35 103 152 151 143 43 118 20 118 198 86 43 237 84 242 56 123 200 158 64 155 41 197 89 46 99 44 230 126 240 58 170 94 104 131 185 105 117 1 1 62 40 0 0 3 0 1 83]
stream-%d.mp4的[]byte是上面的这个

@yapingcat
Copy link
Owner

init.mp4的数据有点问题,你是怎么拿init.mp4数据的?

@zileyuan
Copy link
Author

zileyuan commented Mar 16, 2023

我用之前的方法获取的

func (m *Muxer) bothMuxer(frame *WsFrame) error {
	if frame.codec < mp4.MP4_CODEC_AAC {
		var isVCLNaluType bool
		if frame.codec == mp4.MP4_CODEC_H264 {
			ntype := codec.H264NaluType(frame.raw)
			isVCLNaluType = codec.IsH264VCLNaluType(ntype)
		} else if frame.codec == mp4.MP4_CODEC_H265 {
			ntype := codec.H265NaluType(frame.raw)
			isVCLNaluType = codec.IsH265VCLNaluType(ntype)
		}
		if !isVCLNaluType {
			m.cache = append(m.cache, frame.raw...)
			fmt.Printf("----------!isVCLNaluType\n")
			return errors.New("67899")
		}
		vtid := m.vtid
		fmt.Printf("----------m.vtid %v\n", m.vtid)
		if m.vtid == 0 {
			m.vtid = m.muxer.AddVideoTrack(frame.codec)
		}
		if len(m.cache) > 0 {
			m.cache = append(m.cache, frame.raw...)
			fmt.Printf("----------isVCLNaluType cache %v\n", m.cache)
			m.muxer.Write(m.vtid, m.cache, frame.timestamp, frame.timestamp)
			m.cache = m.cache[:0]
		} else {
			m.muxer.Write(m.vtid, frame.raw, frame.timestamp, frame.timestamp)
		}
		if vtid == 0 {
			b := make([]byte, 4096)
			initSeg := bytes.NewBuffer(b)
			m.muxer.WriteInitSegment(initSeg)
			m.segment = initSeg.Bytes()
			fmt.Printf("----------initSeg.Bytes() %v, len %v\n", initSeg.Bytes(), len(initSeg.Bytes()))
		}
		m.muxer.FlushFragment()

		// m.muxer.WriteTrailer()
	} else {
		return errors.New("12345")
		// tid = muxer.AddAudioTrack(frame.codec, mp4.WithAudioSampleRate(frame.sampleRate), mp4.WithAudioChannelCount(frame.channelCount))
		// if frame.codec == mp4.MP4_CODEC_AAC {
		// 	interval := uint64((1024 * 1000 / frame.sampleRate))
		// 	timestamp := frame.timestamp
		// 	codec.SplitAACFrame(frame.raw, func(bytes []byte) {
		// 		muxer.Write(tid, bytes, timestamp, timestamp)
		// 		timestamp += interval
		// 	})
		// } else if frame.codec == mp4.MP4_CODEC_G711A || frame.codec == mp4.MP4_CODEC_G711U {
		// 	muxer.Write(tid, frame.raw, frame.timestamp, frame.timestamp)
		// }
		// muxer.WriteTrailer()
	}
	return nil
}

@yapingcat
Copy link
Owner

我用之前的方法获取的

func (m *Muxer) bothMuxer(frame *WsFrame) error {
	if frame.codec < mp4.MP4_CODEC_AAC {
		var isVCLNaluType bool
		if frame.codec == mp4.MP4_CODEC_H264 {
			ntype := codec.H264NaluType(frame.raw)
			isVCLNaluType = codec.IsH264VCLNaluType(ntype)
		} else if frame.codec == mp4.MP4_CODEC_H265 {
			ntype := codec.H265NaluType(frame.raw)
			isVCLNaluType = codec.IsH265VCLNaluType(ntype)
		}
		if !isVCLNaluType {
			m.cache = append(m.cache, frame.raw...)
			fmt.Printf("----------!isVCLNaluType\n")
			return errors.New("67899")
		}
		vtid := m.vtid
		fmt.Printf("----------m.vtid %v\n", m.vtid)
		if m.vtid == 0 {
			m.vtid = m.muxer.AddVideoTrack(frame.codec)
		}
		if len(m.cache) > 0 {
			m.cache = append(m.cache, frame.raw...)
			fmt.Printf("----------isVCLNaluType cache %v\n", m.cache)
			m.muxer.Write(m.vtid, m.cache, frame.timestamp, frame.timestamp)
			m.cache = m.cache[:0]
		} else {
			m.muxer.Write(m.vtid, frame.raw, frame.timestamp, frame.timestamp)
		}
		if vtid == 0 {
			b := make([]byte, 4096)
			initSeg := bytes.NewBuffer(b)
			m.muxer.WriteInitSegment(initSeg)
			m.segment = initSeg.Bytes()
			fmt.Printf("----------initSeg.Bytes() %v, len %v\n", initSeg.Bytes(), len(initSeg.Bytes()))
		}
		m.muxer.FlushFragment()

		// m.muxer.WriteTrailer()
	} else {
		return errors.New("12345")
		// tid = muxer.AddAudioTrack(frame.codec, mp4.WithAudioSampleRate(frame.sampleRate), mp4.WithAudioChannelCount(frame.channelCount))
		// if frame.codec == mp4.MP4_CODEC_AAC {
		// 	interval := uint64((1024 * 1000 / frame.sampleRate))
		// 	timestamp := frame.timestamp
		// 	codec.SplitAACFrame(frame.raw, func(bytes []byte) {
		// 		muxer.Write(tid, bytes, timestamp, timestamp)
		// 		timestamp += interval
		// 	})
		// } else if frame.codec == mp4.MP4_CODEC_G711A || frame.codec == mp4.MP4_CODEC_G711U {
		// 	muxer.Write(tid, frame.raw, frame.timestamp, frame.timestamp)
		// }
		// muxer.WriteTrailer()
	}
	return nil
}

b := make([]byte, 4096) 改成 b := make([]byte, 0,4096)

@zileyuan
Copy link
Author

改了make的调用,有效果,现在init 数据是下面
[0 0 0 28 102 116 121 112 105 115 111 53 0 0 2 0 105 115 111 53 105 115 111 54 109 112 52 49 0 0 2 139 109 111 111 118 0 0 0 108 109 118 104 100 0 0 0 0 224 57 25 10 224 57 25 10 0 0 3 232 0 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 64 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 1 239 116 114 97 107 0 0 0 92 116 107 104 100 0 0 0 3 224 57 25 10 224 57 25 10 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 64 0 0 0 2 192 0 0 2 64 0 0 0 0 1 139 109 100 105 97 0 0 0 32 109 100 104 100 0 0 0 0 224 57 25 10 224 57 25 10 0 0 3 232 0 0 0 0 85 196 0 0 0 0 0 45 104 100 108 114 0 0 0 0 0 0 0 0 118 105 100 101 0 0 0 0 0 0 0 0 0 0 0 0 86 105 100 101 111 72 97 110 100 108 101 114 0 0 0 1 54 109 105 110 102 0 0 0 20 118 109 104 100 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 36 100 105 110 102 0 0 0 28 100 114 101 102 0 0 0 0 0 0 0 1 0 0 0 12 117 114 108 32 0 0 0 1 0 0 0 246 115 116 98 108 0 0 0 150 115 116 115 100 0 0 0 0 0 0 0 1 0 0 0 134 97 118 99 49 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 192 2 64 0 72 0 0 0 72 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 24 255 255 0 0 0 48 97 118 99 67 1 77 96 52 255 225 0 25 39 77 96 52 141 141 80 88 9 52 217 0 0 3 0 1 0 0 3 0 50 15 22 38 160 1 0 4 40 238 9 200 0 0 0 16 115 116 116 115 0 0 0 0 0 0 0 0 0 0 0 16 115 116 115 99 0 0 0 0 0 0 0 0 0 0 0 20 115 116 115 122 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 16 115 116 99 111 0 0 0 0 0 0 0 0 0 0 0 20 115 116 115 115 0 0 0 0 0 0 0 1 0 0 0 1 0 0 0 40 109 118 101 120 0 0 0 32 116 114 101 120 0 0 0 0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]

@zileyuan
Copy link
Author

但是我把上面的init数据和后面的m.fws.buffer 的数据传到web的mse,没有错误了,但是video一直在打转,没有画面出现

@zileyuan
Copy link
Author

应该是时间戳的问题

@yapingcat
Copy link
Owner

应该是时间戳的问题

现在能出画面了吗?

@zileyuan
Copy link
Author

第一幅画面出来了,但是一直在推数据,但是画面静止不动

@yapingcat
Copy link
Owner

麻烦更新一下版本,修了一个bug

代码也需要加一个分帧的逻辑

func (m *Muxer) bothMuxer(frame *WsFrame) error {
	if frame.codec < mp4.MP4_CODEC_AAC {
		codec.SplitFrameWithStartCode(frame.raw, func(nalu []byte) bool {
			var isVCLNaluType bool
			if frame.codec == mp4.MP4_CODEC_H264 {
				ntype := codec.H264NaluType(nalu)
				isVCLNaluType = codec.IsH264VCLNaluType(ntype)
			} else if frame.codec == mp4.MP4_CODEC_H265 {
				ntype := codec.H265NaluType(nalu)
				isVCLNaluType = codec.IsH265VCLNaluType(ntype)
			}
			if !isVCLNaluType {
				m.cache = append(m.cache, nalu...)
				fmt.Printf("----------!isVCLNaluType\n")
				return false
			}
			vtid := m.vtid
			fmt.Printf("----------m.vtid %v\n", m.vtid)
			if m.vtid == 0 {
				m.vtid = m.muxer.AddVideoTrack(frame.codec)
			}
			if len(m.cache) > 0 {
				m.cache = append(m.cache, nalu...)
				fmt.Printf("----------isVCLNaluType cache %v\n", m.cache)
				m.muxer.Write(m.vtid, m.cache, frame.timestamp, frame.timestamp)
				m.cache = m.cache[:0]
			} else {
				m.muxer.Write(m.vtid, nalu, frame.timestamp, frame.timestamp)
			}
			if vtid == 0 {
				b := make([]byte, 4096)
				initSeg := bytes.NewBuffer(b)
				m.muxer.WriteInitSegment(initSeg)
				m.segment = initSeg.Bytes()
				fmt.Printf("----------initSeg.Bytes() %v, len %v\n", initSeg.Bytes(), len(initSeg.Bytes()))
			}
			m.muxer.FlushFragment()
			return true
		})

		// m.muxer.WriteTrailer()
	} else {
		return errors.New("12345")
		// tid = muxer.AddAudioTrack(frame.codec, mp4.WithAudioSampleRate(frame.sampleRate), mp4.WithAudioChannelCount(frame.channelCount))
		// if frame.codec == mp4.MP4_CODEC_AAC {
		// 	interval := uint64((1024 * 1000 / frame.sampleRate))
		// 	timestamp := frame.timestamp
		// 	codec.SplitAACFrame(frame.raw, func(bytes []byte) {
		// 		muxer.Write(tid, bytes, timestamp, timestamp)
		// 		timestamp += interval
		// 	})
		// } else if frame.codec == mp4.MP4_CODEC_G711A || frame.codec == mp4.MP4_CODEC_G711U {
		// 	muxer.Write(tid, frame.raw, frame.timestamp, frame.timestamp)
		// }
		// muxer.WriteTrailer()
	}
	return nil
}

@zileyuan
Copy link
Author

我更新了最新的版本,代码改成如下,但是到网页上报错了 MEDIA_ERR_SRC_NOT_SUPPORTED

func (m *Muxer) bothMuxer(frame *WsFrame) {
	if frame.codec < mp4.MP4_CODEC_AAC {
		codec.SplitFrameWithStartCode(frame.raw, func(nalu []byte) bool {
			var isVCLNaluType bool
			if frame.codec == mp4.MP4_CODEC_H264 {
				ntype := codec.H264NaluType(nalu)
				isVCLNaluType = codec.IsH264VCLNaluType(ntype)
			} else if frame.codec == mp4.MP4_CODEC_H265 {
				ntype := codec.H265NaluType(nalu)
				isVCLNaluType = codec.IsH265VCLNaluType(ntype)
			}
			if !isVCLNaluType {
				m.cache = append(m.cache, nalu...)
				fmt.Printf("----------!isVCLNaluType\n")
				return true
			}
			vtid := m.vtid
			fmt.Printf("----------m.vtid %v\n", m.vtid)
			if m.vtid == 0 {
				m.time = frame.timestamp
				m.vtid = m.muxer.AddVideoTrack(frame.codec)
			}
			fmt.Printf("----------frame.timestamp-m.time %v\n", frame.timestamp-m.time)
			if len(m.cache) > 0 {
				m.cache = append(m.cache, nalu...)
				fmt.Printf("----------isVCLNaluType cache %v\n", m.cache)
				m.muxer.Write(m.vtid, m.cache, frame.timestamp-m.time, frame.timestamp-m.time)
				m.cache = m.cache[:0]
			} else {
				m.muxer.Write(m.vtid, nalu, frame.timestamp-m.time, frame.timestamp-m.time)
			}
			if vtid == 0 {
				b := make([]byte, 0, 4096)
				initSeg := bytes.NewBuffer(b)
				m.muxer.WriteInitSegment(initSeg)
				m.segment = initSeg.Bytes()
				fmt.Printf("----------initSeg.Bytes() %v, len %v\n", initSeg.Bytes(), len(initSeg.Bytes()))
			}
			m.muxer.FlushFragment()
			return true
		})
		// m.muxer.WriteTrailer()
	} else {
		// return errors.New("12345")
		// tid = muxer.AddAudioTrack(frame.codec, mp4.WithAudioSampleRate(frame.sampleRate), mp4.WithAudioChannelCount(frame.channelCount))
		// if frame.codec == mp4.MP4_CODEC_AAC {
		// 	interval := uint64((1024 * 1000 / frame.sampleRate))
		// 	timestamp := frame.timestamp
		// 	codec.SplitAACFrame(frame.raw, func(bytes []byte) {
		// 		muxer.Write(tid, bytes, timestamp, timestamp)
		// 		timestamp += interval
		// 	})
		// } else if frame.codec == mp4.MP4_CODEC_G711A || frame.codec == mp4.MP4_CODEC_G711U {
		// 	muxer.Write(tid, frame.raw, frame.timestamp, frame.timestamp)
		// }
		// muxer.WriteTrailer()
	}
}

@zileyuan
Copy link
Author

改成如下代码,和之前一样,画面出来了,但是画面不动,但是数据一直在推送

func (m *Muxer) bothMuxer(frame *WsFrame) {
	if frame.codec < mp4.MP4_CODEC_AAC {
		// codec.SplitFrameWithStartCode(frame.raw, func(nalu []byte) bool {
		nalu := frame.raw
		var isVCLNaluType bool
		if frame.codec == mp4.MP4_CODEC_H264 {
			ntype := codec.H264NaluType(nalu)
			isVCLNaluType = codec.IsH264VCLNaluType(ntype)
		} else if frame.codec == mp4.MP4_CODEC_H265 {
			ntype := codec.H265NaluType(nalu)
			isVCLNaluType = codec.IsH265VCLNaluType(ntype)
		}
		if !isVCLNaluType {
			m.cache = append(m.cache, nalu...)
			fmt.Printf("----------!isVCLNaluType\n")
			// return true
			return
		}
		vtid := m.vtid
		fmt.Printf("----------m.vtid %v\n", m.vtid)
		if m.vtid == 0 {
			m.time = frame.timestamp
			m.vtid = m.muxer.AddVideoTrack(frame.codec)
		}
		fmt.Printf("----------frame.timestamp-m.time %v\n", frame.timestamp-m.time)
		if len(m.cache) > 0 {
			m.cache = append(m.cache, nalu...)
			fmt.Printf("----------isVCLNaluType cache %v\n", m.cache)
			m.muxer.Write(m.vtid, m.cache, frame.timestamp-m.time, frame.timestamp-m.time)
			m.cache = m.cache[:0]
		} else {
			m.muxer.Write(m.vtid, nalu, frame.timestamp-m.time, frame.timestamp-m.time)
		}
		if vtid == 0 {
			b := make([]byte, 0, 4096)
			initSeg := bytes.NewBuffer(b)
			m.muxer.WriteInitSegment(initSeg)
			m.segment = initSeg.Bytes()
			fmt.Printf("----------initSeg.Bytes() %v, len %v\n", initSeg.Bytes(), len(initSeg.Bytes()))
		}
		m.muxer.FlushFragment()
		// return true
		return
		// })
		// m.muxer.WriteTrailer()
	} else {
		// return errors.New("12345")
		// tid = muxer.AddAudioTrack(frame.codec, mp4.WithAudioSampleRate(frame.sampleRate), mp4.WithAudioChannelCount(frame.channelCount))
		// if frame.codec == mp4.MP4_CODEC_AAC {
		// 	interval := uint64((1024 * 1000 / frame.sampleRate))
		// 	timestamp := frame.timestamp
		// 	codec.SplitAACFrame(frame.raw, func(bytes []byte) {
		// 		muxer.Write(tid, bytes, timestamp, timestamp)
		// 		timestamp += interval
		// 	})
		// } else if frame.codec == mp4.MP4_CODEC_G711A || frame.codec == mp4.MP4_CODEC_G711U {
		// 	muxer.Write(tid, frame.raw, frame.timestamp, frame.timestamp)
		// }
		// muxer.WriteTrailer()
	}
}

@yapingcat
Copy link
Owner

尝试一下把所有的数据写入一个mp4文件,然后用浏览器和ffplay测试播放

@yapingcat
Copy link
Owner

yapingcat commented Mar 17, 2023

加分帧逻辑,播放报错问题,已经解决 0cf3fb8

尝试一下把所有的数据写入一个mp4文件,然后用浏览器和ffplay测试播放

有条件的话,可以把这个MP4文件发我看一下。

@zileyuan
Copy link
Author

又更新了一下代码,可以播放了,可能还是时间戳的问题,画面会停一会,过一会就正常了

@zileyuan
Copy link
Author

隔了几天了,😄,我把AAC音频加上去, Video出错“CHUNK_DEMUXER_ERROR_APPEND_FAILED: Initialization segment misses expected aac track.”

type Muxer struct {
	cache    []byte
	init     []byte
	segments [][]byte
	fws      *fmp4WriterSeeker
	muxer    *mp4.Movmuxer
	first    bool
	vtid     uint32
	atid     uint32
}

func (m *Muxer) doBothMuxer(frame *WsFrame) {
	m.bothMuxer(frame)
}

func fixTimestamp(timestamp uint64) uint64 {
	t := time.UnixMilli(int64(timestamp))
	x := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local)
	return timestamp - uint64(x.UnixMilli())
}

func (m *Muxer) reBindWriter() {
	segment := m.fws.buffer
	// fmt.Printf("----------segment %v\n", segment)
	m.segments = append(m.segments, segment)
	fws := newFmp4WriterSeeker(1024 * 1024)
	m.muxer.ReBindWriter(fws)
	m.fws = fws
}

func (m *Muxer) initSegment() {
	b := make([]byte, 0, 4096)
	initSeg := bytes.NewBuffer(b)
	m.muxer.WriteInitSegment(initSeg)
	m.init = initSeg.Bytes()
}

func (m *Muxer) bothMuxer(frame *WsFrame) {
	if frame.codec < mp4.MP4_CODEC_AAC {
		codec.SplitFrameWithStartCode(frame.raw, func(nalu []byte) bool {
			var isVCLNaluType bool
			if frame.codec == mp4.MP4_CODEC_H264 {
				ntype := codec.H264NaluType(nalu)
				isVCLNaluType = codec.IsH264VCLNaluType(ntype)
			} else if frame.codec == mp4.MP4_CODEC_H265 {
				ntype := codec.H265NaluType(nalu)
				isVCLNaluType = codec.IsH265VCLNaluType(ntype)
			}
			if !isVCLNaluType {
				m.cache = append(m.cache, nalu...)
				// fmt.Printf("----------!isVCLNaluType\n")
				return true
			}
			// fmt.Printf("----------m.vtid %v\n", m.vtid)
			if m.vtid == 0 {
				m.vtid = m.muxer.AddVideoTrack(frame.codec)
			}
			newTimestamp := fixTimestamp(frame.timestamp)
			// fmt.Printf("----------newTimestamp %v\n", newTimestamp)
			if len(m.cache) > 0 {
				m.cache = append(m.cache, nalu...)
				// fmt.Printf("----------isVCLNaluType cache %v\n", m.cache)
				m.muxer.Write(m.vtid, m.cache, newTimestamp, newTimestamp)
				m.cache = m.cache[:0]
			} else {
				m.muxer.Write(m.vtid, nalu, newTimestamp, newTimestamp)
			}
			if m.first {
				m.initSegment()
				m.first = false
			}
			m.muxer.FlushFragment()
			m.reBindWriter()
			return true
		})
	} else {
		if m.atid == 0 {
			fmt.Printf("Add Trace %v\n", frame.codec)
			// sampleRate = 8000 channel = 1
			m.atid = m.muxer.AddAudioTrack(frame.codec, mp4.WithAudioSampleRate(8000), mp4.WithAudioChannelCount(1))
		}
		newTimestamp := fixTimestamp(frame.timestamp)
		interval := uint64((1024 * 1000 / 8000))
		if frame.codec == mp4.MP4_CODEC_AAC {
			codec.SplitAACFrame(frame.raw, func(bytes []byte) {
				fmt.Printf("aac newTimestamp %v", newTimestamp)
				m.muxer.Write(m.atid, bytes, newTimestamp, newTimestamp)
				newTimestamp += interval

			})
		} else if frame.codec == mp4.MP4_CODEC_G711A || frame.codec == mp4.MP4_CODEC_G711U {
			m.muxer.Write(m.atid, frame.raw, newTimestamp, newTimestamp)
		}
		if m.first {
			m.initSegment()
			m.first = false
		}
		m.muxer.FlushFragment()
		m.reBindWriter()
	}
}

@zileyuan
Copy link
Author

我查了一下 确实是AAC的音频,也进入了SplitAACFrame 函数

@yapingcat
Copy link
Owner

在调用m.initSegment() 之前必须先调用AddAudioTrack和AddVideoTrack,这样生成的init segment里面才会包含音视频的track信息
从代码上看m.initSegment() 之前,只会有音频或者视频一个track

@yapingcat
Copy link
Owner

yapingcat commented Mar 20, 2023

我建议是写入一帧音频之后再生成init segment,这样写起来比较简单

否则, 你需要参考下面这段代码

gomedia/go-mp4/mp4track.go

Lines 427 to 445 in 0cf3fb8

if aacextra.asc == nil || len(aacextra.asc) <= 0 {
asc, err := codec.ConvertADTSToASC(aacframes)
if err != nil {
return err
}
aacextra.asc = asc.Encode()
if track.chanelCount == 0 {
track.chanelCount = asc.Channel_configuration
}
if track.sampleRate == 0 {
track.sampleRate = uint32(codec.AACSampleIdxToSample(int(asc.Sample_freq_index)))
}
if track.sampleBits == 0 {
// aac has no fixed bit depth, so we just set it to the default of 16
// see AudioSampleEntry (stsd-box) and https://superuser.com/a/1173507
track.sampleBits = 16
}
}

拿到asc ,chanelCount,sampleRate,sampleBits,之后在AddTrack的时候 使用WithExtraData,WithAudioChannelCount 等选项写入这些参数

@zileyuan
Copy link
Author

我改了一下,web 不报错了 但是video 播放器一直在打转,我在想是不是把sourcebuffer 分开两个 一个做音频,一个做视频,分别塞入音频和视频数据

@yapingcat yapingcat mentioned this issue Aug 31, 2023
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

2 participants