Decoder Test HEVC

#include "common/bitstream.h"
#include "common/VPS.h"
#include "decoder/CAVLC_decoder.h"
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <sstream>
#include <vector>
static const unsigned char bytestream_vps[] =
{
     0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x00,
     0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0xba, 0x94, 0x90, 0x24
};
static const unsigned char bytestream_sps[] =
{
     0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00,
     0x03, 0x00, 0x00, 0x03, 0x00, 0xba, 0xa0, 0x0d, 0x08, 0x0f, 0x1f, 0xe5, 0x94,
     0x99, 0x24, 0x6d, 0x86, 0x16, 0x22, 0xaa, 0x4c, 0x4c, 0x32, 0xfb, 0x3e, 0xbc,
     0xdf, 0x96, 0x7d, 0x78, 0x51, 0x18, 0x9c, 0xbb, 0x20
};
static const unsigned char bytestream_pps[] =
{
     0x44, 0x01, 0xc1, 0xa5, 0x58, 0x11, 0x20
};
static const unsigned char rbsp_vps[] =
{
     0x0c, 0x01, 0xff, 0xff, 0x01, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
     0x00, 0x00, 0xba, 0x94, 0x90, 0x24
};
static const unsigned char rbsp_sps[] =
{
     0x01, 0x01, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xba,
     0xa0, 0x0d, 0x08, 0x0f, 0x1f, 0xe5, 0x94, 0x99, 0x24, 0x6d, 0x86, 0x16, 0x22,
     0xaa, 0x4c, 0x4c, 0x32, 0xfb, 0x3e, 0xbc, 0xdf, 0x96, 0x7d, 0x78, 0x51, 0x18,
     0x9c, 0xbb, 0x20
};

static const unsigned char rbsp_pps[] =
{
  int main(int argc, const char* argv[])
  {
      std::stringstream stream_vps;
      std::stringstream stream_sps;
     std::stringstream stream_pps;

for(int i=0; i<19; i++) stream_vps.put(rbsp_vps[i]);
for(int i=0; i<42; i++) stream_sps.put(rbsp_sps[i]);
for(int i=0; i<5; i++) stream_pps.put(rbsp_pps[i]);

HEVC::VPS vps;
HEVC::CAVLCDecoder decoder;
const unsigned char* buf =
reinterpret_cast<const unsigned char*>(stream_vps.str().c_str());
HEVC::InputBitstream bs(buf, stream_vps.str().length());
decoder.decode_vps(&bs, &vps);
assert(vps.get_vps_id() == 0);
assert(vps.get_max_layers() == 1);
assert(vps.get_max_sub_layers() == 1);
assert(vps.get_temporal_id_nesting_flag());
assert(vps.get_sub_layer_ordering_info_present_flag());
assert(vps.get_max_layer_id()==0);
assert(vps.get_num_layer_sets()==1);
assert(!vps.get_timing_info_present_flag());
assert(!vps.get_extension_flag());
0xc1, 0xa5, 0x58, 0x11, 0x20
};

return EXIT_SUCCESS;
}

Gstreamer

Why we care about Gstreamer?

GStreamer is a pipeline based multimedia framework written in the C programming language with the type system based on GObject. GStreamer allows a programmer to create a variety of media-handling components, including simple audio playback, audio and video playback, recording, streaming, and editing. The pipeline design serves as a base to create many types of multimedia applications such as video editors, streaming media broadcasters, and media players.

Gstreamer Tutorials "Audio, Video Multimedia Toolkit"
GStreamer is a multimedia framework written in the C programming language with the type system based on GObject."
http://gstreamer.freedesktop.org/
http://gstreamer.freedesktop.org/documentation/
http://gstreamer.freedesktop.org/src/

Application Development Manual:

[HTML] http://gstreamer.freedesktop.org/data/doc/gstreamer/head/manual/html/index.html

[PDF] http://gstreamer.freedesktop.org/data/doc/gstreamer/stable/manual/manual.pdf

Plugin Writer's Guide :

[HTML] http://gstreamer.freedesktop.org/data/doc/gstreamer/head/pwg/html/index.html

[PDF] http://gstreamer.freedesktop.org/data/doc/gstreamer/stable/pwg/pwg.pdf


GStreamer Quick Hello World type Tutorial

[HTML] Part1: http://bluwiki.com/go/GStreamer

[HTML] Part2: http://bluwiki.com/go/GStreamer/C_Hello_World


GStreamer Plugin Development Video

* An introduction to GStreamer terms, usage and the GStreamer pipeline model.

* Construction of simple pipelines using available GStreamer elements, building a video decoding pipeline

* Look at the requirements for various types of processing elements: Data sources, sinks, filters, converters.

* Development of a simple custom plugin for processing raw video frames

* Extending the custom element in various ways, to render video effects, provide custom communication with the application.

Participants should be familiar with GObject/GLib, as GStreamer code relies heavily on both.

[HTML] http://lca2007.linux.org.au/talk/272.html

[VIDEO] http://mirror.linux.org.au/pub/linux.conf.au/2007/video/talks/272.ogg

HEVC NAL UNIT

//Standard way of adding this Method for NAL processing:

#include "NAL.h"
#include "common/bitstream.h"
#include "common/syntax_element.h"
#include <cassert>
#include <iterator>
#include <list>
namespace HEVC
{
    using namespace SyntaxElement;
 

void NALUnitHeader::read_header(InputBitstream* bs)
{
assert(bs!=NULL);
unsigned int code;
code = read_flag(bs);
assert(code==0); // forbidden zero bit
_nal_unit_type = static_cast<NAL_UNIT_TYPE>(read_code(bs, 6));
_nuh_layer_id = read_code(bs, 6);
_nuh_temporal_id = read_code(bs, 3) - 1;
}
void NALUnitHeader::write_header(OutputBitstream* bs)
{
assert(bs!=NULL);
write_flag(bs, 0); // forbidden zero bit
write_code(bs, static_cast<unsigned int>(_nal_unit_type), 6);
write_code(bs, _nuh_layer_id, 6);
write_code(bs, _nuh_temporal_id + 1, 3);
}
void NALUnit::read_rbsp()
{
unsigned char* ix_beg = _rbsp_bytes.data();
unsigned char* ix_end = ix_beg + _rbsp_bytes.size();
std::list<std::vector<unsigned char>::iterator> positions;
for(unsigned char* ix = ix_beg+3; ix<ix_end;)
{
static const int shift[4] = {2, 4, 4, 1};
switch(*ix)
{
case 0x00:
case 0x01:
case 0x02:
case 0x03:
if(*(ix-3)==0x00 && *(ix-2)==0x00 && *(ix-1)==0x03)
{
std::vector<unsigned char>::iterator it = _rbsp_bytes.begin();
std::advance(it, ix-ix_beg-1);
positions.push_back(it);
ix++;
break;
}
ix+=shift[*ix];
break;
default:
ix+=4;
break;
}
}

for(auto it=positions.rbegin(); it!=positions.rend(); it++)
{
_rbsp_bytes.erase(*it);
}
}
void NALUnit::write_rbsp()
{
if(_rbsp_bytes.size() < 3) return;
for(auto it=_rbsp_bytes.begin()+2; it!=_rbsp_bytes.end();)
{
switch(*it)
{
case 0x00:
case 0x01:
case 0x02:
case 0x03:
if(*(it-1)==0x00 && *(it-2)==0x00)
{
_rbsp_bytes.insert(it, 0x03);
}

it++;
break;
default:
if(std::distance(it, _rbsp_bytes.end())>=3)
{
std::advance(it, 3);
break;
}
return;
}

}

}
void NALUnit::read()
{

{

InputBitstream bs(_rbsp_bytes.data(), _rbsp_bytes.size());
read_header(&bs);
_rbsp_bytes.erase(_rbsp_bytes.begin(), _rbsp_bytes.begin()+2);
}
read_rbsp();
}
void NALUnit::write()
{
write_rbsp();
{
OutputBitstream bs;
write_header(&bs);
_rbsp_bytes.insert(_rbsp_bytes.begin(),
bs.get_stream().cbegin(), bs.get_stream().cend());
}

}

} // HEVC

MPEG-DASH with VLC

Frames Capture and Video Creation Using Xuggler

The concept is to open the media file, loop through a specific video stream and at specific intervals capture the corresponding frame, convert it to an image and dump the binary contents into a file. Here is what the code for all these looks like:

	package com.javacodegeeks.xuggler;

	import java.awt.image.BufferedImage;
	import java.io.File;

	import java.io.IOException;

	import javax.imageio.ImageIO;

	import com.xuggle.mediatool.IMediaReader;
	import com.xuggle.mediatool.MediaListenerAdapter;

	import com.xuggle.mediatool.ToolFactory;
	import com.xuggle.mediatool.event.IVideoPictureEvent;

	import com.xuggle.xuggler.Global;

	public class VideoThumbnailsExample {

	public static final double SECONDS_BETWEEN_FRAMES = 10;

	private static final String inputFilename = "c:/Java_is_Everywhere.mp4";
	private static final String outputFilePrefix = "c:/snapshots/mysnapshot";

	// The video stream index, used to ensure we display frames from one and

	// only one video stream from the media container.
	private static int mVideoStreamIndex = -1;

	// Time of last frame write

	private static long mLastPtsWrite = Global.NO_PTS;

	public static final long MICRO_SECONDS_BETWEEN_FRAMES =
	(long)(Global.DEFAULT_PTS_PER_SECOND * SECONDS_BETWEEN_FRAMES);

	public static void main(String[] args) {

	IMediaReader mediaReader = ToolFactory.makeReader(inputFilename);

	// stipulate that we want BufferedImages created in BGR 24bit color space

	mediaReader.setBufferedImageTypeToGenerate(BufferedImage.TYPE_3BYTE_BGR);

	mediaReader.addListener(new ImageSnapListener());

	// read out the contents of the media file and
	// dispatch events to the attached listener

	while (mediaReader.readPacket() == null) ;

	}

	private static class ImageSnapListener extends MediaListenerAdapter {

	public void onVideoPicture(IVideoPictureEvent event) {

	if (event.getStreamIndex() != mVideoStreamIndex) {
	// if the selected video stream id is not yet set, go ahead an

	// select this lucky video stream
	if (mVideoStreamIndex == -1)

	mVideoStreamIndex = event.getStreamIndex();
	// no need to show frames from this video stream

	else
	return;

	}

	// if uninitialized, back date mLastPtsWrite to get the very first frame
	if (mLastPtsWrite == Global.NO_PTS)

	mLastPtsWrite = event.getTimeStamp() - MICRO_SECONDS_BETWEEN_FRAMES;

	// if it's time to write the next frame
	if (event.getTimeStamp() - mLastPtsWrite >=

	MICRO_SECONDS_BETWEEN_FRAMES) {

	String outputFilename = dumpImageToFile(event.getImage());

	// indicate file written
	double seconds = ((double) event.getTimeStamp()) /

	Global.DEFAULT_PTS_PER_SECOND;
	System.out.printf(

	"at elapsed time of %6.3f seconds wrote: %s\n",
	seconds, outputFilename);

	// update last write time

	mLastPtsWrite += MICRO_SECONDS_BETWEEN_FRAMES;
	}

	}

	private String dumpImageToFile(BufferedImage image) {

	try {
	String outputFilename = outputFilePrefix +

	System.currentTimeMillis() + ".png";
	ImageIO.write(image, "png", new File(outputFilename));

	return outputFilename;
	}

	catch (IOException e) {
	e.printStackTrace();

	return null;
	}

	}

	}

}

This might seem a bit overwhelming, but it is really quite straightforward. Let me provide some details for you. We start by creating anIMediaReader from an input file. The media reader is used to read and decode media. Since we wish to manipulate the captures video frames as images, we use the setBufferedImageTypeToGenerate method to denote this. The reader opens up a media container, reads packets from it, decodes the data, and then dispatches information about the data to any registered IMediaListener objects. Here is where our custom class, ImageSnapListener, comes into play.

Our listener extends the MediaListenerAdapter, which is an adapter (provides empty methods) implementing the IMediaListener interface. Objects that implement this interface are notified about events generated during the video processing. We only care about handling video events, thus we only implement the IMediaListener.onVideoPicture method. Inside that we use the provided IVideoPictureEvent object to find what stream (video only) we are dealing with.

Since we wish to capture frames in specific times, we have to mess a little with timestamps. First, we make sure we handle the very first frame by checking against the value of the Global.NO_PTS constant, which is a value that means no time stamp is set for a given object. Then, if the minimum elapsed time has passes, we capture the frame by invoking the IVideoPictureEvent.getImage method, which returns the underlying BufferedImage. Note that we are talking about elapsed video time and not “real time”. We then dump the image data to a file in PNG format using the ImageIO.write utility method. Finally, we update the last write time.

Let’s run this application in order to see the results. As input file, I am using an old Sun commercial proclaiming that “Java is Everywhere”. I have downloaded locally the MP4 version provided. Here is what the output console will look like:

at elapsed time of 0.000 seconds wrote: c:/snapshots/mysnapshot1298228503292.png
at elapsed time of 10.010 seconds wrote: c:/snapshots/mysnapshot1298228504014.png
at elapsed time of 20.020 seconds wrote: c:/snapshots/mysnapshot1298228504463.png
…
at elapsed time of 130.063 seconds wrote: c:/snapshots/mysnapshot1298228509454.png
at elapsed time of 140.007 seconds wrote: c:/snapshots/mysnapshot1298228509933.png
at elapsed time of 150.017 seconds wrote: c:/snapshots/mysnapshot1298228510379.png

The total video time is about 151 seconds so we capture 16 frames.


In order to create video, we will have to take a bit more low level approach in comparison to the MediaTool API that we have seen so far. Don’t worry though, it is not going to be complicated. The main idea is that we create a media writer, add some stream information to it, encode our media (the screenshot images), and close the writer. Let’s see the code used to achieve this:

	package com.javacodegeeks.xuggler;

	import java.awt.AWTException;
	import java.awt.Dimension;

	import java.awt.Rectangle;
	import java.awt.Robot;

	import java.awt.Toolkit;
	import java.awt.image.BufferedImage;

	import java.util.concurrent.TimeUnit;

	import com.xuggle.mediatool.IMediaWriter;
	import com.xuggle.mediatool.ToolFactory;

	import com.xuggle.xuggler.ICodec;

	public class ScreenRecordingExample {

	private static final double FRAME_RATE = 50;

	private static final int SECONDS_TO_RUN_FOR = 20;

	private static final String outputFilename = "c:/mydesktop.mp4";

	private static Dimension screenBounds;

	public static void main(String[] args) {

	// let's make a IMediaWriter to write the file.
	final IMediaWriter writer = ToolFactory.makeWriter(outputFilename);

	screenBounds = Toolkit.getDefaultToolkit().getScreenSize();

	// We tell it we're going to add one video stream, with id 0,

	// at position 0, and that it will have a fixed frame rate of FRAME_RATE.
	writer.addVideoStream(0, 0, ICodec.ID.CODEC_ID_MPEG4,

	screenBounds.width/2, screenBounds.height/2);

	long startTime = System.nanoTime();

	for (int index = 0; index < SECONDS_TO_RUN_FOR * FRAME_RATE; index++) {

	// take the screen shot
	BufferedImage screen = getDesktopScreenshot();

	// convert to the right image type

	BufferedImage bgrScreen = convertToType(screen,
	BufferedImage.TYPE_3BYTE_BGR);

	// encode the image to stream #0

	writer.encodeVideo(0, bgrScreen, System.nanoTime() - startTime,
	TimeUnit.NANOSECONDS);

	// sleep for frame rate milliseconds

	try {
	Thread.sleep((long) (1000 / FRAME_RATE));

	}
	catch (InterruptedException e) {

	// ignore
	}

	}

	// tell the writer to close and write the trailer if  needed

	writer.close();

	}

	public static BufferedImage convertToType(BufferedImage sourceImage, int targetType) {

	BufferedImage image;

	// if the source image is already the target type, return the source image
	if (sourceImage.getType() == targetType) {

	image = sourceImage;
	}

	// otherwise create a new image of the target type and draw the new image
	else {

	image = new BufferedImage(sourceImage.getWidth(),
	sourceImage.getHeight(), targetType);

	image.getGraphics().drawImage(sourceImage, 0, 0, null);
	}

	return image;

	}

	private static BufferedImage getDesktopScreenshot() {

	try {
	Robot robot = new Robot();

	Rectangle captureSize = new Rectangle(screenBounds);
	return robot.createScreenCapture(captureSize);

	}
	catch (AWTException e) {

	e.printStackTrace();
	return null;

	}

	}

}

We start by creating an IMediaWriter from a given output file. This class encodes and decodes media, handling both audio and video streams. Xuggler guesses the output format from the file name extension (in our case MP4) and sets some default values appropriately. We then use the addVideoStream method to add a new video stream, providing its index, the codec type used (MPEG-4 here) and the video dimensions. The dimensions are set equal to half of the screen’s dimensions in this example.

Then we execute a loop that runs for a number of times equal to the desired frame rate multiplied with the desired running time. Inside the loop, we generate a screen snapshot as described in the Java2D: Screenshots with Java article. We retrieve the screenshot as aBufferedImage and convert it to the appropriate type (TYPE_3BYTE_BGR) if it is not already there.

Next, we encode the image to the video stream using the IMediaWriter.encodeVideo method. We provide the stream index, the image, the elapsed video time and the time unit. Then, we sleep for the appropriate number of time, depending on the desired frame rate. When the loop is over, we close the writer and write the trailer if necessary, depending on the video format (this is done automatically by Xuggler).

If we execute the application, a video will be created which has recorded your desktop actions.