Skip to content

H264/MP4 Mux/Demux Fixes #218

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nickgaray merged 10 commits into from
Jul 24, 2026
Merged

H264/MP4 Mux/Demux Fixes #218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
5fc0667
Remove debug code, fix mp4 muxer
kyle-fitzp Jun 10, 2026
bf9e612
Use BSF for extradata processing, remove manual extradata injection/c…
kyle-fitzp Jun 11, 2026
1e3f2a1
Fix open codec logic
kyle-fitzp Jun 17, 2026
f17e3f5
Merge branch 'master' into mp4-mux-fixes
kyle-fitzp Jun 17, 2026
cba48d1
Remove unused imports and logger
kyle-fitzp Jun 17, 2026
f506d5d
Check for Annex B before dumping extradata
kyle-fitzp Jul 1, 2026
30a52ca
Fixed check for parameter sets
kyle-fitzp Jul 1, 2026
5c5e691
Use BSF filter list, filter nal units
kyle-fitzp Jul 21, 2026
ac8b70c
Remove commented code, document filter_units
kyle-fitzp Jul 23, 2026
32cbf6e
Remove unused methods
kyle-fitzp Jul 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions ...ors/video/sensorhub-driver-ffmpeg/src/main/java/org/sensorhub/mpegts/MpegTsProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,9 @@ public void closeStream() {
logger.debug("closeStream");

if (streamOpened) {
videoStreamContext.close();
audioStreamContext.close();
dataStreamContext.close();

synchronized (avFormatContextLock) {
if (avFormatContext != null) {
Expand Down
244 changes: 96 additions & 148 deletions sensors/video/sensorhub-driver-ffmpeg/src/main/java/org/sensorhub/mpegts/StreamContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,13 @@
package org.sensorhub.mpegts;

import org.bytedeco.ffmpeg.avcodec.AVCodec;
import org.bytedeco.ffmpeg.avcodec.AVCodecContext;
import org.bytedeco.ffmpeg.avcodec.AVCodecParameters;
import org.bytedeco.ffmpeg.avcodec.AVPacket;
import org.bytedeco.ffmpeg.avcodec.*;
import org.bytedeco.ffmpeg.avformat.AVFormatContext;
import org.bytedeco.ffmpeg.global.avcodec;
import org.bytedeco.javacpp.BytePointer;
import org.bytedeco.javacpp.PointerPointer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.ByteArrayOutputStream;

public class StreamContext {

private static final Logger logger = LoggerFactory.getLogger(StreamContext.class);
Expand Down Expand Up @@ -43,10 +38,14 @@ public class StreamContext {

private int codecId;

private byte[] extraData = null;
private AVBSFContext bsfContext = null;

private boolean isInjectingExtradata = false;

private volatile boolean isOpen = false;

private final Object lock = new Object();
Comment thread
nickgaray marked this conversation as resolved.

/**
* Returns the ID of the stream associated with this context.
*
Expand Down Expand Up @@ -132,40 +131,72 @@ public boolean hasStream() {
return streamId != INVALID_STREAM_ID;
}

private String selectBsfName(int codecId, AVFormatContext avFormatContext) {

// Best guess at useful BSFs. If a video format does not work, may
// need to add a corresponding BSF here.
return switch (codecId) {
// filter_units=pass_types=1|5: Allow IDR and non-IDR slices to pass. All other units are removed. Parameter sets are added in-stream with dump_extra.
case avcodec.AV_CODEC_ID_H264 -> "h264_mp4toannexb,filter_units=pass_types=1|5,dump_extra";
// filter_units=pass_types=0-31: Allow slices to pass. All other units are removed. Parameter sets are added in-stream with dump_extra.
case avcodec.AV_CODEC_ID_HEVC -> "hevc_mp4toannexb,filter_units=pass_types=0-31,dump_extra";
case avcodec.AV_CODEC_ID_VVC -> "vvc_mp4toannexb,dump_extra";
case avcodec.AV_CODEC_ID_EVC -> "evc_mp4toannexb,dump_extra";
case avcodec.AV_CODEC_ID_MPEG4 -> "mpeg4_unpack_bframes,dump_extra";
case avcodec.AV_CODEC_ID_MPEG2VIDEO, avcodec.AV_CODEC_ID_VC1, avcodec.AV_CODEC_ID_WMV3 -> "dump_extra";
case avcodec.AV_CODEC_ID_MJPEG -> "mjpeg2jpeg";
case avcodec.AV_CODEC_ID_VP9 -> "vp9_superframe_split";
case avcodec.AV_CODEC_ID_AV1 -> "av1_frame_split";
default -> null;
};
}

/**
* Opens the codec context, and sets it up according to the {@link StreamContext#streamId}.
* This method must be called before any packets are decoded.
*
* @throws IllegalStateException if the codec is unsupported or cannot be opened.
*/
public void openCodecContext(AVFormatContext avFormatContext) throws IllegalStateException {
if (!hasStream()) return;
synchronized (lock) {
if (isOpen) return;
if (!hasStream()) return;

AVCodecParameters params = avFormatContext.streams(getStreamId()).codecpar();
AVCodecParameters params = avFormatContext.streams(getStreamId()).codecpar();

// Get the associated codec from the ID stored in the context
AVCodec codec = avcodec.avcodec_find_decoder(params.codec_id());
// Get the associated codec from the ID stored in the context
AVCodec codec = avcodec.avcodec_find_decoder(params.codec_id());

if (codec == null) {
throw new IllegalStateException("Unsupported codec");
}
if (codec == null) {
throw new IllegalStateException("Unsupported codec");
}

// Store the codec name
setCodecName(codec.name().getString());
setCodecId(codec.id());

if (isInjectingExtradata) {
BytePointer extra = params.extradata();
int extraLen = params.extradata_size();
if (codecId == avcodec.AV_CODEC_ID_H264) {
extraData = getAnnexBExtradata(extra, extraLen);
} else if (codecId == avcodec.AV_CODEC_ID_HEVC) {
extraData = getHvccAnnexBExtradata(extra, extraLen);
} else {
extraData = null;
// Store the codec name
setCodecName(codec.name().getString());
setCodecId(codec.id());

if (isInjectingExtradata) {
String bsfNames = selectBsfName(codecId, avFormatContext);

// Initialize BSFs if needed
if (bsfNames != null) {
bsfContext = new AVBSFContext(null);
if (avcodec.av_bsf_list_parse_str(bsfNames, bsfContext) < 0) {
throw new IllegalStateException("Failed to parse BSF list: " + bsfNames);

}
if (avcodec.avcodec_parameters_copy(bsfContext.par_in(), params) < 0) {
throw new IllegalStateException("Failed to copy codec parameters");
}

bsfContext.time_base_in(avFormatContext.streams(getStreamId()).time_base());

if (avcodec.av_bsf_init(bsfContext) < 0) {
throw new IllegalStateException("Failed to initialize BSF: " + bsfNames);
}
}
}
} else {
extraData = null;
isOpen = true;
}
}

Expand All @@ -176,134 +207,51 @@ public void openCodecContext(AVFormatContext avFormatContext) throws IllegalStat
* @param avPacket The packet to process
*/
public void processPacket(AVPacket avPacket) {
if (getStreamId() == INVALID_STREAM_ID) return;
if (getDataBufferListener() == null) return;
if (avPacket.stream_index() != getStreamId()) return;

// Extract the data buffer from the packet
byte[] dataBuffer = new byte[avPacket.size()];
avPacket.data().get(dataBuffer);

// Prepend cached parameter sets ahead of every keyframe so late-join
// decoders have the VPS/SPS/PPS they need. Applies to both H.264 (SPS/PPS)
// and HEVC (VPS/SPS/PPS) when extradata injection is enabled.
if (extraData != null && (avPacket.flags() & avcodec.AV_PKT_FLAG_KEY) != 0) {
getDataBufferListener().onDataBuffer(new DataBufferRecord(avPacket.pts() * getStreamTimeBase(), extraData));
}
// Pass data buffer to the interested listener
getDataBufferListener().onDataBuffer(new DataBufferRecord(avPacket.pts() * getStreamTimeBase(), dataBuffer));
}

/**
* Converts the given H.264 extradata from AVCC format (AVCDecoderConfigurationRecord,
* ISO/IEC 14496-15 §5.2.4.1) to Annex B format. If the data is already in Annex B format,
* it is returned directly.
*
* @param extradata A BytePointer containing the codec extradata. Must not be null.
* @param size The size of the extradata in bytes.
* @return A byte array containing the Annex B formatted extradata, or {@code null} if the data
* is invalid, cannot be processed, or if an error occurs during processing.
*/
private byte[] getAnnexBExtradata(BytePointer extradata, int size) {
if (extradata == null || size < 7) return null;

byte[] data = new byte[size];
extradata.get(data);

// Check for Annex B start code, either 0x000001 or 0x00000001
if (data[0] == 0x00 && data[1] == 0x00 && (data[2] == 0x01 || (data[2] == 0x00 && data[3] == 0x01))) {
// Already in Annex B format, use directly
return data;
}

// Otherwise, we need to convert AVCC to Annex B format
ByteArrayOutputStream out = new ByteArrayOutputStream();
int pos = 5;
int numSps = data[pos++] & 0x1F;
try {
for (int i = 0; i < numSps; i++) {
if (pos + 2 > data.length) return null;
int spsLen = ((data[pos++] & 0xFF) << 8) | (data[pos++] & 0xFF);
if (pos + spsLen > data.length) return null;
out.write(new byte[]{0x00, 0x00, 0x00, 0x01});
out.write(data, pos, spsLen);
pos += spsLen;
}

int numPps = data[pos++] & 0xFF;
for (int i = 0; i < numPps; i++) {
if (pos + 2 > data.length) return null;
int ppsLen = ((data[pos++] & 0xFF) << 8) | (data[pos++] & 0xFF);
if (pos + ppsLen > data.length) return null;
out.write(new byte[]{0x00, 0x00, 0x00, 0x01});
out.write(data, pos, ppsLen);
pos += ppsLen;
synchronized (lock) {
if (!isOpen) return;
if (getStreamId() == INVALID_STREAM_ID) return;
if (getDataBufferListener() == null) return;
if (avPacket.stream_index() != getStreamId()) return;

if (bsfContext != null) {
AVPacket filtered = avcodec.av_packet_alloc();
Comment thread
kyle-fitzp marked this conversation as resolved.
AVPacket clonePacket = avcodec.av_packet_clone(avPacket);
try {
avcodec.av_bsf_send_packet(bsfContext, clonePacket);
while (avcodec.av_bsf_receive_packet(bsfContext, filtered) >= 0) {
notifyPacketListener(filtered);
avcodec.av_packet_unref(filtered);
}
} finally {
avcodec.av_packet_free(clonePacket);
avcodec.av_packet_free(filtered);
}
} else {
// Pass packet data straight to listener
notifyPacketListener(avPacket);
}
} catch (Exception e) {
logger.error("Error extracting SPS and PPS from AVCC extradata", e);
}
return out.toByteArray();
}

/**
* Converts HEVC extradata from HVCC format (HEVCDecoderConfigurationRecord,
* ISO/IEC 14496-15 §8.3.3.1.2) to Annex B format. Walks the {@code numOfArrays}
* table and emits every contained NAL (typically VPS=32, SPS=33, PPS=34) prefixed
* with the 4-byte 0x00000001 start code so that downstream pipelines can decode
* without out-of-band parameter sets.
* <p>
* If the data is already in Annex B format, it is returned directly.
*
* @param extradata A BytePointer containing the codec extradata. Must not be null.
* @param size The size of the extradata in bytes.
* @return A byte array containing the Annex B formatted VPS/SPS/PPS NALs, or
* {@code null} if the data is invalid, too short, or cannot be parsed.
* Notify the listener with the data buffer extracted from the packet.
* @param avPacket The packet containing the data buffer
*/
private byte[] getHvccAnnexBExtradata(BytePointer extradata, int size) {
// HVCC has a 22-byte fixed header followed by numOfArrays (1 byte),
// so the minimum useful size is 23 bytes.
if (extradata == null || size < 23) return null;

byte[] data = new byte[size];
extradata.get(data);

// If already Annex B, pass straight through.
if (data[0] == 0x00 && data[1] == 0x00 && (data[2] == 0x01 || (data[2] == 0x00 && data[3] == 0x01))) {
return data;
}

// configurationVersion must be 1 per ISO/IEC 14496-15.
if ((data[0] & 0xFF) != 1) {
logger.warn("Unsupported HVCC configurationVersion: {}", data[0] & 0xFF);
return null;
}

ByteArrayOutputStream out = new ByteArrayOutputStream();
try {
int pos = 22;
int numOfArrays = data[pos++] & 0xFF;

for (int i = 0; i < numOfArrays; i++) {
if (pos + 3 > data.length) return null;
private void notifyPacketListener(AVPacket avPacket) {
byte[] dataBuffer = new byte[avPacket.size()];
avPacket.data().get(dataBuffer);
getDataBufferListener().onDataBuffer(new DataBufferRecord(avPacket.pts() * getStreamTimeBase(), dataBuffer));
}

pos++;
int numNalus = ((data[pos++] & 0xFF) << 8) | (data[pos++] & 0xFF);
public void close() {
synchronized (lock) {
if (!isOpen) return;
isOpen = false;

for (int j = 0; j < numNalus; j++) {
if (pos + 2 > data.length) return null;
int nalLen = ((data[pos++] & 0xFF) << 8) | (data[pos++] & 0xFF);
if (nalLen <= 0 || pos + nalLen > data.length) return null;
out.write(new byte[]{0x00, 0x00, 0x00, 0x01});
out.write(data, pos, nalLen);
pos += nalLen;
}
if (bsfContext != null) {
avcodec.av_bsf_free(bsfContext);
bsfContext = null;
}
} catch (Exception e) {
logger.error("Error extracting VPS/SPS/PPS from HVCC extradata", e);
return null;
}

byte[] result = out.toByteArray();
return result.length == 0 ? null : result;
}
}
2 changes: 1 addition & 1 deletion ...orhub-service-video/src/main/java/org/sensorhub/impl/service/sos/video/MP4Serializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ public void sendNextFrame(DataBlock nextFrame, OutputStream os) throws IOExcepti
ByteBuffer nals = ByteBuffer.wrap(frameData);

// debug
os.write(frameData);
//os.write(frameData);
//os.flush();

// look for next nal unit
Expand Down
Loading