24#include <juce_audio_basics/juce_audio_basics.h>
25#include <juce_audio_devices/juce_audio_devices.h>
27#include <QApplication>
45Frame::Frame(int64_t number,
int width,
int height, std::string color,
int samples,
int channels)
46 : audio(std::make_shared<
juce::AudioBuffer<float>>(channels, samples)),
47 number(number), width(width), height(height),
48 pixel_ratio(1,1), color(color),
51 has_audio_data(false), has_image_data(false),
62Frame::Frame(int64_t number,
int width,
int height, std::string color)
63 :
Frame::
Frame(number, width, height, color, 0, 2) {}
67 :
Frame::
Frame(number, 1, 1,
"#000000", samples, channels) {}
90 channels = other.channels;
92 height = other.height;
93 channel_layout = other.channel_layout;
96 sample_rate = other.sample_rate;
97 pixel_ratio =
Fraction(other.pixel_ratio.
num, other.pixel_ratio.
den);
99 max_audio_sample = other.max_audio_sample;
102 image = std::make_shared<QImage>(*(other.image));
104 audio = std::make_shared<juce::AudioBuffer<float>>(*(other.
audio));
105 if (other.wave_image)
106 wave_image = std::make_shared<QImage>(*(other.wave_image));
122 if (!QApplication::instance()) {
125 static char* argv[1] = {NULL};
126 previewApp = std::make_shared<QApplication>(argc, argv);
130 std::shared_ptr<QImage> previewImage =
GetImage();
133 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
136 previewImage = std::make_shared<QImage>(previewImage->scaled(
137 previewImage->size().width(), previewImage->size().height() * pixel_ratio.
Reciprocal().
ToDouble(),
138 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
142 QWidget previewWindow;
143 previewWindow.setStyleSheet(
"background-color: #000000;");
148 previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
149 previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
150 layout.addWidget(&previewLabel);
153 previewWindow.setLayout(&layout);
154 previewWindow.show();
159std::shared_ptr<QImage>
Frame::GetWaveform(
int width,
int height,
int Red,
int Green,
int Blue,
int Alpha)
165 QVector<QPointF> lines;
166 QVector<QPointF> labels;
170 if (total_samples > 0)
173 int new_height = 200 *
audio->getNumChannels();
174 int height_padding = 20 * (
audio->getNumChannels() - 1);
175 int total_height = new_height + height_padding;
177 float zero_height = 1.0;
181 for (
int channel = 0; channel <
audio->getNumChannels(); channel++)
186 const float *samples =
audio->getReadPointer(channel);
191 float value = samples[sample] * 100.0;
195 if (value > -zero_height && value < 0.0) {
196 value = -zero_height;
197 }
else if (value > 0.0 && value < zero_height) {
202 lines.push_back(QPointF(X, Y));
203 lines.push_back(QPointF(X, Y - value));
207 labels.push_back(QPointF(5.0, Y - 5.0));
210 Y += (200 + height_padding);
215 wave_image = std::make_shared<QImage>(
216 total_width, total_height, QImage::Format_RGBA8888_Premultiplied);
217 wave_image->fill(QColor(0,0,0,0));
220 QPainter painter(wave_image.get());
224 pen.setColor(QColor(Red, Green, Blue, Alpha));
226 pen.setStyle(Qt::SolidLine);
230 painter.drawLines(lines);
236 wave_image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
237 wave_image->fill(QColor(QString::fromStdString(
"#000000")));
241 if (wave_image->width() != width || wave_image->height() != height) {
242 QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
243 wave_image = std::make_shared<QImage>(scaled_wave_image);
261 wave_image =
GetWaveform(width, height, Red, Green, Blue, Alpha);
264 return wave_image->constBits();
273 if (!QApplication::instance()) {
276 static char* argv[1] = {NULL};
277 previewApp = std::make_shared<QApplication>(argc, argv);
281 QWidget previewWindow;
282 previewWindow.setStyleSheet(
"background-color: #000000;");
287 previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
288 previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
289 layout.addWidget(&previewLabel);
292 previewWindow.setLayout(&layout);
293 previewWindow.show();
305 return audio->getMagnitude(channel, sample, magnitude_range);
309 return audio->getMagnitude(sample, magnitude_range);
320 return buffer->getWritePointer(channel);
329 float *output = NULL;
330 int num_of_channels =
audio->getNumChannels();
334 output =
new float[num_of_channels * num_of_samples];
338 for (
int sample = 0; sample < num_of_samples; sample++)
340 for (
int channel = 0; channel < num_of_channels; channel++)
343 output[position] = buffer->getReadPointer(channel)[sample];
351 *sample_count = num_of_samples;
360 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
362 return audio->getNumChannels();
370 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
371 return max_audio_sample;
382 int64_t total_bytes = 0;
384 total_bytes +=
static_cast<int64_t
>(
385 width * height *
sizeof(char) * 4);
389 total_bytes += (sample_rate / 24.0) *
sizeof(
float);
405 return image->constBits();
417 return image->constScanLine(row);
421bool Frame::CheckPixel(
int row,
int col,
int red,
int green,
int blue,
int alpha,
int threshold) {
422 int col_pos = col * 4;
423 if (!image || row < 0 || row >= (height - 1) ||
424 col_pos < 0 || col_pos >= (width - 1) ) {
429 const unsigned char* pixels =
GetPixels(row);
430 if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
431 pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
432 pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
433 pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
445 pixel_ratio.
num = num;
446 pixel_ratio.
den = den;
462 double previous_samples = (sample_rate * fps_rate) * (
number - 1);
463 double previous_samples_remainder = fmod(previous_samples, (
double)channels);
464 previous_samples -= previous_samples_remainder;
467 double total_samples = (sample_rate * fps_rate) *
number;
468 double total_samples_remainder = fmod(total_samples, (
double)channels);
469 total_samples -= total_samples_remainder;
473 int samples_per_frame = round(total_samples - previous_samples);
474 if (samples_per_frame < 0)
475 samples_per_frame = 0;
476 return samples_per_frame;
506 return channel_layout;
511void Frame::Save(std::string path,
float scale, std::string format,
int quality)
514 std::shared_ptr<QImage> previewImage =
GetImage();
517 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
520 previewImage = std::make_shared<QImage>(previewImage->scaled(
521 previewImage->size().width(), previewImage->size().height() * pixel_ratio.
Reciprocal().
ToDouble(),
522 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
526 if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
529 previewImage = std::make_shared<QImage>(previewImage->scaled(
530 previewImage->size().width() * scale, previewImage->size().height() * scale,
531 Qt::KeepAspectRatio, Qt::SmoothTransformation));
535 previewImage->save(QString::fromStdString(path), format.c_str(), quality);
539void Frame::Thumbnail(std::string path,
int new_width,
int new_height, std::string mask_path, std::string overlay_path,
540 std::string background_color,
bool ignore_aspect, std::string format,
int quality,
float rotate) {
543 auto thumbnail = std::make_shared<QImage>(
544 new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
545 thumbnail->fill(QColor(QString::fromStdString(background_color)));
548 QPainter painter(thumbnail.get());
549 painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing,
true);
552 std::shared_ptr<QImage> previewImage =
GetImage();
555 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
558 int aspect_width = previewImage->size().width();
559 int aspect_height = previewImage->size().height() * pixel_ratio.
Reciprocal().
ToDouble();
562 previewImage = std::make_shared<QImage>(previewImage->scaled(
563 aspect_width, aspect_height,
564 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
570 previewImage = std::make_shared<QImage>(previewImage->scaled(
571 new_width, new_height,
572 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
575 previewImage = std::make_shared<QImage>(previewImage->scaled(
576 new_width, new_height,
577 Qt::KeepAspectRatio, Qt::SmoothTransformation));
580 int x = (new_width - previewImage->size().width()) / 2.0;
581 int y = (new_height - previewImage->size().height()) / 2.0;
582 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
586 QTransform transform;
587 float origin_x = previewImage->width() / 2.0;
588 float origin_y = previewImage->height() / 2.0;
589 transform.translate(origin_x, origin_y);
590 transform.rotate(rotate);
591 transform.translate(-origin_x,-origin_y);
592 painter.setTransform(transform);
595 painter.drawImage(x, y, *previewImage);
599 if (overlay_path !=
"") {
601 auto overlay = std::make_shared<QImage>();
602 overlay->load(QString::fromStdString(overlay_path));
605 overlay = std::make_shared<QImage>(
606 overlay->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
609 overlay = std::make_shared<QImage>(overlay->scaled(
610 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
613 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
614 painter.drawImage(0, 0, *overlay);
619 if (mask_path !=
"") {
621 auto mask = std::make_shared<QImage>();
622 mask->load(QString::fromStdString(mask_path));
625 mask = std::make_shared<QImage>(
626 mask->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
629 mask = std::make_shared<QImage>(mask->scaled(
630 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
633 mask->invertPixels();
636 unsigned char *pixels =
static_cast<unsigned char *
>(thumbnail->bits());
637 const unsigned char *mask_pixels =
static_cast<const unsigned char *
>(mask->constBits());
641 for (
int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
644 int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
645 int Frame_Alpha = pixels[byte_index + 3];
646 int Mask_Value = constrain(Frame_Alpha - gray_value);
649 pixels[byte_index + 3] = Mask_Value;
658 thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
662int Frame::constrain(
int color_value)
667 else if (color_value > 255)
675 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
680 AddColor(QColor(QString::fromStdString(new_color)));
687 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
688 image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
691 image->fill(new_color);
697 int new_width,
int new_height,
int bytes_per_pixel,
698 QImage::Format type,
const unsigned char *pixels_)
706 auto new_image = std::make_shared<QImage>(
708 new_width, new_height,
709 new_width * bytes_per_pixel,
711 (QImageCleanupFunction) &openshot::cleanUpBuffer,
725 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
729 if (image->format() != QImage::Format_RGBA8888_Premultiplied)
730 *image = image->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
733 width = image->width();
734 height = image->height();
753 if (image == new_image || image->size() != new_image->size()) {
756 else if (new_image->format() != QImage::Format_RGBA8888_Premultiplied) {
757 new_image = std::make_shared<QImage>(
758 new_image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
765 const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
766 unsigned char *pixels = image->bits();
767 const unsigned char *new_pixels = new_image->constBits();
774 for (
int row = start; row < image->height(); row += 2) {
775 int offset = row * image->bytesPerLine();
776 memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
780 height = image->height();
781 width = image->width();
790 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
793 audio->setSize(channels, length,
true,
true,
false);
794 channel_layout = layout;
798 max_audio_sample = length;
804 if (
audio && !audio_reversed) {
807 audio_reversed =
true;
812void Frame::AddAudio(
bool replaceSamples,
int destChannel,
int destStartSample,
const float* source,
int numSamples,
float gainToApplyToSource = 1.0f) {
813 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
816 int destStartSampleAdjusted = max(destStartSample, 0);
819 int new_length = destStartSampleAdjusted + numSamples;
820 int new_channel_length =
audio->getNumChannels();
821 if (destChannel >= new_channel_length)
822 new_channel_length = destChannel + 1;
823 if (new_length >
audio->getNumSamples() || new_channel_length >
audio->getNumChannels())
824 audio->setSize(new_channel_length, new_length,
true,
true,
false);
828 audio->clear(destChannel, destStartSampleAdjusted, numSamples);
831 audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
835 if (new_length > max_audio_sample)
836 max_audio_sample = new_length;
839 audio_reversed =
false;
843void Frame::ApplyGainRamp(
int destChannel,
int destStartSample,
int numSamples,
float initial_gain = 0.0f,
float final_gain = 1.0f)
845 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
848 audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
867 cv::Mat mat = cv::Mat(qimage->height(), qimage->width(), CV_8UC4, (uchar*)qimage->constBits(), qimage->bytesPerLine()).clone();
868 cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 );
869 int from_to[] = { 0,0, 1,1, 2,2 };
870 cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 );
871 cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR);
891 cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
892 QImage qimg((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888);
894 std::shared_ptr<QImage> imgIn = std::make_shared<QImage>(qimg.copy());
897 if (imgIn->format() != QImage::Format_RGBA8888_Premultiplied)
898 *imgIn = imgIn->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
918 juce::AudioDeviceManager deviceManager;
919 juce::String error = deviceManager.initialise (
926 if (error.isNotEmpty()) {
927 cout <<
"Error on initialise(): " << error << endl;
930 juce::AudioSourcePlayer audioSourcePlayer;
931 deviceManager.addAudioCallback (&audioSourcePlayer);
933 std::unique_ptr<AudioBufferSource> my_source;
937 juce::TimeSliceThread my_thread(
"Audio buffer thread");
940 my_thread.startThread();
942 juce::AudioTransportSource transport1;
943 transport1.setSource (my_source.get(),
946 (
double) sample_rate,
947 audio->getNumChannels());
948 transport1.setPosition (0);
949 transport1.setGain(1.0);
953 juce::MixerAudioSource mixer;
954 mixer.addInputSource(&transport1,
false);
955 audioSourcePlayer.setSource (&mixer);
960 while (transport1.isPlaying())
962 cout <<
"playing" << endl;
963 std::this_thread::sleep_for(std::chrono::seconds(1));
966 cout <<
"DONE!!!" << endl;
969 transport1.setSource (0);
970 audioSourcePlayer.setSource (0);
971 my_thread.stopThread(500);
972 deviceManager.removeAudioCallback (&audioSourcePlayer);
973 deviceManager.closeAudioDevice();
974 deviceManager.removeAllChangeListeners();
975 deviceManager.dispatchPendingMessages();
977 cout <<
"End of Play()" << endl;
985 const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
988 audio->setSize(channels, numSamples,
false,
true,
false);
993 max_audio_sample = numSamples;
996 audio_reversed =
false;
Header file for AudioBufferSource class.
Header file for AudioResampler class.
Header file for Frame class.
Header file for QtUtilities (compatibiity overlay)
This class is used to expose an AudioBuffer<float> as an AudioSource in JUCE.
This class represents a fraction.
int num
Numerator for the fraction.
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
int den
Denominator for the fraction.
This class represents a single frame of video (i.e. image & audio data)
Frame & operator=(const Frame &other)
Assignment operator.
std::shared_ptr< juce::AudioBuffer< float > > audio
std::shared_ptr< QImage > Mat2Qimage(cv::Mat img)
Convert OpenCV Mat to QImage.
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
const unsigned char * GetPixels()
Get pixel data (as packets)
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG,...
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
int GetAudioChannelsCount()
Get number of audio channels.
cv::Mat Qimage2mat(std::shared_ptr< QImage > &qimage)
Convert Qimage to Mat.
bool has_image_data
This frame has been loaded with pixel data.
int GetWidth()
Get height of image.
int SampleRate()
Get the original sample rate of this frame's audio data.
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
cv::Mat GetImageCV()
Get pointer to OpenCV Mat image object.
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
void Play()
Play audio samples for this frame.
float * GetInterleavedAudioSamples(int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate.
bool has_audio_data
This frame has been loaded with audio data.
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
openshot::ChannelLayout ChannelsLayout()
void AddAudioSilence(int numSamples)
Add audio silence.
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
void DisplayWaveform()
Display the wave form.
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
float * GetAudioSamples(int channel)
Get an array of sample data (and optional reverse the sample values)
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
virtual ~Frame()
Destructor.
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Frame()
Constructor - blank frame.
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
int GetAudioSamplesCount()
Get number of audio samples.
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
int GetHeight()
Get height of image.
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
void SetFrameNumber(int64_t number)
Set frame number.
juce::AudioBuffer< float > * GetAudioSampleBuffer()
int64_t number
This is the frame number (starting at 1)
void SetImageCV(cv::Mat _image)
Set pointer to OpenCV image object.
This namespace is the default namespace for all code in the openshot library.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...