Frame.h
Go to the documentation of this file.
/**
* @file
* @brief Header file for Frame class
* @author Jonathan Thomas <jonathan@openshot.org>
*
* @ref License
*/
/* LICENSE
*
* Copyright (c) 2008-2019 OpenShot Studios, LLC
* <http://www.openshotstudios.com/>. This file is part of
* OpenShot Library (libopenshot), an open-source project dedicated to
* delivering high quality video editing and animation solutions to the
* world. For more information visit <http://www.openshot.org/>.
*
* OpenShot Library (libopenshot) is free software: you can redistribute it
* and/or modify it under the terms of the GNU Lesser General Public License
* as published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* OpenShot Library (libopenshot) is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef OPENSHOT_FRAME_H
#define OPENSHOT_FRAME_H
#include <iomanip>
#include <sstream>
#include <queue>
#include <QtWidgets/QApplication>
#include <QtGui/QImage>
#include <QtGui/QColor>
#include <QtGui/QBitmap>
#include <QtCore/QString>
#include <QtCore/QVector>
#include <QtGui/QPainter>
#include <QtWidgets/QHBoxLayout>
#include <QtWidgets/QWidget>
#include <QtWidgets/QLabel>
#include <memory>
#include <unistd.h>
#include "ZmqLogger.h"
#include "ChannelLayouts.h"
#include "AudioBufferSource.h"
#include "AudioResampler.h"
#include "Fraction.h"
#include "JuceHeader.h"
#ifdef USE_IMAGEMAGICK
#include "MagickUtilities.h"
#endif
namespace openshot
{
/**
* @brief This class represents a single frame of video (i.e. image & audio data)
*
* FileReaders (such as FFmpegReader) use instances of this class to store the individual frames of video,
* which include both the image data (i.e. pixels) and audio samples. An openshot::Frame also has many debug
* methods, such as the ability to display the image (using X11), play the audio samples (using JUCE), or
* display the audio waveform as an image.
*
* FileWriters (such as FFmpegWriter) use instances of this class to create new video files, image files, or
* video streams. So, think of these openshot::Frame instances as the smallest unit of work in a video
* editor.
*
* There are many ways to create an instance of an openshot::Frame:
* @code
*
* // Most basic: a blank frame (300x200 blank image, 48kHz audio silence)
* Frame();
*
* // Image only settings (48kHz audio silence)
* Frame(1, // Frame number
* 720, // Width of image
* 480, // Height of image
* "#000000" // HTML color code of background color
* );
*
* // Audio only (300x200 blank image)
* Frame(number, // Frame number
* 44100, // Sample rate of audio stream
* 2 // Number of audio channels
* );
*
* // Image and Audio settings (user defines all key settings)
* Frame(number, // Frame number
* 720, // Width of image
* 480, // Height of image
* "#000000" // HTML color code of background color
* 44100, // Sample rate of audio stream
* 2 // Number of audio channels
* );
*
* // Some methods require a shared pointer to an openshot::Frame object.
* std::shared_ptr<Frame> f(new Frame(1, 720, 480, "#000000", 44100, 2));
*
* @endcode
*/
class Frame
{
private:
std::shared_ptr<QImage> image;
std::shared_ptr<QImage> wave_image;
std::shared_ptr<juce::AudioSampleBuffer> audio;
std::shared_ptr<QApplication> previewApp;
juce::CriticalSection addingImageSection;
juce::CriticalSection addingAudioSection;
const unsigned char *qbuffer;
openshot::Fraction pixel_ratio;
int channels;
ChannelLayout channel_layout;
int width;
int height;
int sample_rate;
std::string color;
int64_t max_audio_sample; ///< The max audio sample count added to this frame
/// Constrain a color value from 0 to 255
int constrain(int color_value);
public:
int64_t number; ///< This is the frame number (starting at 1)
bool has_audio_data; ///< This frame has been loaded with audio data
bool has_image_data; ///< This frame has been loaded with pixel data
/// Constructor - blank frame (300x200 blank image, 48kHz audio silence)
Frame();
/// Constructor - image only (48kHz audio silence)
Frame(int64_t number, int width, int height, std::string color);
/// Constructor - audio only (300x200 blank image)
Frame(int64_t number, int samples, int channels);
/// Constructor - image & audio
Frame(int64_t number, int width, int height, std::string color, int samples, int channels);
/// Copy constructor
Frame ( const Frame &other );
/// Assignment operator
Frame& operator= (const Frame& other);
/// Destructor
virtual ~Frame();
/// Add (or replace) pixel data to the frame (based on a solid color)
void AddColor(int new_width, int new_height, std::string new_color);
/// Add (or replace) pixel data to the frame
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_);
/// Add (or replace) pixel data to the frame
void AddImage(std::shared_ptr<QImage> new_image);
/// Add (or replace) pixel data to the frame (for only the odd or even lines)
void AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines);
#ifdef USE_IMAGEMAGICK
/// Add (or replace) pixel data to the frame from an ImageMagick Image
void AddMagickImage(std::shared_ptr<Magick::Image> new_image);
#endif
/// Add audio samples to a specific channel
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource);
/// Add audio silence
void AddAudioSilence(int numSamples);
/// Apply gain ramp (i.e. fading volume)
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain);
/// Channel Layout of audio samples. A frame needs to keep track of this, since Writers do not always
/// know the original channel layout of a frame's audio samples (i.e. mono, stereo, 5 point surround, etc...)
openshot::ChannelLayout ChannelsLayout();
// Set the channel layout of audio samples (i.e. mono, stereo, 5 point surround, etc...)
void ChannelsLayout(openshot::ChannelLayout new_channel_layout) { channel_layout = new_channel_layout; };
/// Clean up buffer after QImage is deleted
static void cleanUpBuffer(void *info);
/// Clear the waveform image (and deallocate its memory)
void ClearWaveform();
/// Copy data and pointers from another Frame instance
void DeepCopy(const Frame& other);
/// Display the frame image to the screen (primarily used for debugging reasons)
void Display();
/// Display the wave form
void DisplayWaveform();
/// Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
float GetAudioSample(int channel, int sample, int magnitude_range);
/// Get an array of sample data
float* GetAudioSamples(int channel);
/// Get an array of sample data (all channels interleaved together), using any sample rate
float* GetInterleavedAudioSamples(int new_sample_rate, openshot::AudioResampler* resampler, int* sample_count);
// Get a planar array of sample data, using any sample rate
float* GetPlanarAudioSamples(int new_sample_rate, openshot::AudioResampler* resampler, int* sample_count);
/// Get number of audio channels
int GetAudioChannelsCount();
/// Get number of audio samples
int GetAudioSamplesCount();
juce::AudioSampleBuffer *GetAudioSampleBuffer();
/// Get the size in bytes of this frame (rough estimate)
int64_t GetBytes();
/// Get pointer to Qt QImage image object
std::shared_ptr<QImage> GetImage();
#ifdef USE_IMAGEMAGICK
/// Get pointer to ImageMagick image object
std::shared_ptr<Magick::Image> GetMagickImage();
#endif
/// Set Pixel Aspect Ratio
openshot::Fraction GetPixelRatio() { return pixel_ratio; };
/// Get pixel data (as packets)
const unsigned char* GetPixels();
/// Get pixel data (for only a single scan-line)
const unsigned char* GetPixels(int row);
/// Check a specific pixel color value (returns True/False)
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold);
/// Get height of image
int GetHeight();
/// Calculate the # of samples per video frame (for the current frame number)
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels);
/// Calculate the # of samples per video frame (for a specific frame number and frame rate)
static int GetSamplesPerFrame(int64_t frame_number, openshot::Fraction fps, int sample_rate, int channels);
/// Get an audio waveform image
std::shared_ptr<QImage> GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha);
/// Get an audio waveform image pixels
const unsigned char* GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha);
/// Get height of image
int GetWidth();
/// Resize audio container to hold more (or less) samples and channels
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout);
/// Get the original sample rate of this frame's audio data
int SampleRate();
/// Set the original sample rate of this frame's audio data
void SampleRate(int orig_sample_rate) { sample_rate = orig_sample_rate; };
/// Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM
void Save(std::string path, float scale, std::string format="PNG", int quality=100);
/// Set frame number
void SetFrameNumber(int64_t number);
/// Set Pixel Aspect Ratio
void SetPixelRatio(int num, int den);
/// Thumbnail the frame image with tons of options to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG).
/// This method allows for masks, overlays, background color, and much more accurate resizing (including padding and centering)
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path,
std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0);
/// Play audio samples for this frame
void Play();
};
}
#endif
AudioBufferSource.h
Header file for AudioBufferSource class.
AudioResampler.h
Header file for AudioResampler class.
ChannelLayouts.h
Header file for ChannelLayout class.
Fraction.h
Header file for Fraction class.
MagickUtilities.h
Header file for MagickUtilities (IM6/IM7 compatibility overlay)
ZmqLogger.h
Header file for ZeroMQ-based Logger class.
openshot::AudioResampler
This class is used to resample audio data for many sequential frames.
Definition: AudioResampler.h:46
openshot::Fraction
This class represents a fraction.
Definition: Fraction.h:45
openshot::Frame
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:107
openshot::Frame::operator=
Frame & operator=(const Frame &other)
Assignment operator.
Definition: Frame.cpp:96
openshot::Frame::cleanUpBuffer
static void cleanUpBuffer(void *info)
Clean up buffer after QImage is deleted.
Definition: Frame.cpp:1045
openshot::Frame::AddColor
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition: Frame.cpp:733
openshot::Frame::GetInterleavedAudioSamples
float * GetInterleavedAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate.
Definition: Frame.cpp:387
openshot::Frame::GetPixels
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition: Frame.cpp:469
openshot::Frame::GetWaveform
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition: Frame.cpp:176
openshot::Frame::GetMagickImage
std::shared_ptr< Magick::Image > GetMagickImage()
Get pointer to ImageMagick image object.
Definition: Frame.cpp:925
openshot::Frame::Save
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG,...
Definition: Frame.cpp:578
openshot::Frame::Display
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition: Frame.cpp:135
openshot::Frame::GetAudioChannelsCount
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:432
openshot::Frame::has_image_data
bool has_image_data
This frame has been loaded with pixel data.
Definition: Frame.h:131
openshot::Frame::GetWidth
int GetWidth()
Get height of image.
Definition: Frame.cpp:559
openshot::Frame::GetPixelRatio
openshot::Fraction GetPixelRatio()
Set Pixel Aspect Ratio.
Definition: Frame.h:235
openshot::Frame::SampleRate
int SampleRate()
Get the original sample rate of this frame's audio data.
Definition: Frame.cpp:565
openshot::Frame::ResizeAudio
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:860
openshot::Frame::DeepCopy
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition: Frame.cpp:105
openshot::Frame::GetPlanarAudioSamples
float * GetPlanarAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Definition: Frame.cpp:341
openshot::Frame::AddMagickImage
void AddMagickImage(std::shared_ptr< Magick::Image > new_image)
Add (or replace) pixel data to the frame from an ImageMagick Image.
Definition: Frame.cpp:949
openshot::Frame::ClearWaveform
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
Definition: Frame.cpp:271
openshot::Frame::Play
void Play()
Play audio samples for this frame.
Definition: Frame.cpp:975
openshot::Frame::SampleRate
void SampleRate(int orig_sample_rate)
Set the original sample rate of this frame's audio data.
Definition: Frame.h:271
openshot::Frame::has_audio_data
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:130
openshot::Frame::GetBytes
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition: Frame.cpp:454
openshot::Frame::ChannelsLayout
openshot::ChannelLayout ChannelsLayout()
Definition: Frame.cpp:571
openshot::Frame::AddAudioSilence
void AddAudioSilence(int numSamples)
Add audio silence.
Definition: Frame.cpp:1056
openshot::Frame::Thumbnail
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
Definition: Frame.cpp:609
openshot::Frame::DisplayWaveform
void DisplayWaveform()
Display the wave form.
Definition: Frame.cpp:288
openshot::Frame::GetAudioSampleBuffer
juce::AudioSampleBuffer * GetAudioSampleBuffer()
Definition: Frame.cpp:448
openshot::Frame::CheckPixel
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
Definition: Frame.cpp:488
openshot::Frame::GetAudioSamples
float * GetAudioSamples(int channel)
Get an array of sample data.
Definition: Frame.cpp:334
openshot::Frame::GetAudioSample
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
Definition: Frame.cpp:321
openshot::Frame::~Frame
virtual ~Frame()
Destructor.
Definition: Frame.cpp:128
openshot::Frame::GetSamplesPerFrame
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:547
openshot::Frame::GetImage
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition: Frame.cpp:913
openshot::Frame::ChannelsLayout
void ChannelsLayout(openshot::ChannelLayout new_channel_layout)
Definition: Frame.h:186
openshot::Frame::Frame
Frame()
Constructor - blank frame (300x200 blank image, 48kHz audio silence)
Definition: Frame.cpp:37
openshot::Frame::AddImage
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:754
openshot::Frame::ApplyGainRamp
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition: Frame.cpp:904
openshot::Frame::GetAudioSamplesCount
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:442
openshot::Frame::AddAudio
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:874
openshot::Frame::GetWaveformPixels
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition: Frame.cpp:278
openshot::Frame::GetHeight
int GetHeight()
Get height of image.
Definition: Frame.cpp:553
openshot::Frame::SetPixelRatio
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition: Frame.cpp:510
openshot::Frame::SetFrameNumber
void SetFrameNumber(int64_t number)
Set frame number.
Definition: Frame.cpp:517
openshot::Frame::number
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:129
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: AudioBufferSource.h:39
openshot::ChannelLayout
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
Definition: ChannelLayouts.h:47