OpenShot Library | libopenshot 0.3.2
Loading...
Searching...
No Matches
FFmpegReader.cpp
Go to the documentation of this file.
1
12// Copyright (c) 2008-2019 OpenShot Studios, LLC, Fabrice Bellard
13//
14// SPDX-License-Identifier: LGPL-3.0-or-later
15
16#include <thread> // for std::this_thread::sleep_for
17#include <chrono> // for std::chrono::milliseconds
18#include <unistd.h>
19
20#include "FFmpegUtilities.h"
21
22#include "FFmpegReader.h"
23#include "Exceptions.h"
24#include "Timeline.h"
25#include "ZmqLogger.h"
26
27#define ENABLE_VAAPI 0
28
29#if USE_HW_ACCEL
30#define MAX_SUPPORTED_WIDTH 1950
31#define MAX_SUPPORTED_HEIGHT 1100
32
33#if ENABLE_VAAPI
34#include "libavutil/hwcontext_vaapi.h"
35
36typedef struct VAAPIDecodeContext {
37 VAProfile va_profile;
38 VAEntrypoint va_entrypoint;
39 VAConfigID va_config;
40 VAContextID va_context;
41
42#if FF_API_STRUCT_VAAPI_CONTEXT
43 // FF_DISABLE_DEPRECATION_WARNINGS
44 int have_old_context;
45 struct vaapi_context *old_context;
46 AVBufferRef *device_ref;
47 // FF_ENABLE_DEPRECATION_WARNINGS
48#endif
49
50 AVHWDeviceContext *device;
51 AVVAAPIDeviceContext *hwctx;
52
53 AVHWFramesContext *frames;
54 AVVAAPIFramesContext *hwfc;
55
56 enum AVPixelFormat surface_format;
57 int surface_count;
58 } VAAPIDecodeContext;
59#endif // ENABLE_VAAPI
60#endif // USE_HW_ACCEL
61
62
63using namespace openshot;
64
65int hw_de_on = 0;
66#if USE_HW_ACCEL
67 AVPixelFormat hw_de_av_pix_fmt_global = AV_PIX_FMT_NONE;
68 AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE;
69#endif
70
71FFmpegReader::FFmpegReader(const std::string &path, bool inspect_reader)
72 : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0), NO_PTS_OFFSET(-99999),
73 path(path), is_video_seek(true), check_interlace(false), check_fps(false), enable_seek(true), is_open(false),
74 seek_audio_frame_found(0), seek_video_frame_found(0),is_duration_known(false), largest_frame_processed(0),
75 current_video_frame(0), packet(NULL), max_concurrent_frames(OPEN_MP_NUM_PROCESSORS), audio_pts(0),
76 video_pts(0), pFormatCtx(NULL), videoStream(-1), audioStream(-1), pCodecCtx(NULL), aCodecCtx(NULL),
77 pStream(NULL), aStream(NULL), pFrame(NULL), previous_packet_location{-1,0},
78 hold_packet(false) {
79
80 // Initialize FFMpeg, and register all formats and codecs
83
84 // Init timestamp offsets
85 pts_offset_seconds = NO_PTS_OFFSET;
86 video_pts_seconds = NO_PTS_OFFSET;
87 audio_pts_seconds = NO_PTS_OFFSET;
88
89 // Init cache
90 working_cache.SetMaxBytesFromInfo(max_concurrent_frames * info.fps.ToDouble() * 2, info.width, info.height, info.sample_rate, info.channels);
92
93 // Open and Close the reader, to populate its attributes (such as height, width, etc...)
94 if (inspect_reader) {
95 Open();
96 Close();
97 }
98}
99
101 if (is_open)
102 // Auto close reader if not already done
103 Close();
104}
105
106// This struct holds the associated video frame and starting sample # for an audio packet.
107bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount) {
108 // Is frame even close to this one?
109 if (abs(location.frame - frame) >= 2)
110 // This is too far away to be considered
111 return false;
112
113 // Note that samples_per_frame can vary slightly frame to frame when the
114 // audio sampling rate is not an integer multiple of the video fps.
115 int64_t diff = samples_per_frame * (location.frame - frame) + location.sample_start - sample_start;
116 if (abs(diff) <= amount)
117 // close
118 return true;
119
120 // not close
121 return false;
122}
123
124#if USE_HW_ACCEL
125
126// Get hardware pix format
127static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts)
128{
129 const enum AVPixelFormat *p;
130
131 for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) {
132 switch (*p) {
133#if defined(__unix__)
134 // Linux pix formats
135 case AV_PIX_FMT_VAAPI:
136 hw_de_av_pix_fmt_global = AV_PIX_FMT_VAAPI;
137 hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VAAPI;
138 return *p;
139 break;
140 case AV_PIX_FMT_VDPAU:
141 hw_de_av_pix_fmt_global = AV_PIX_FMT_VDPAU;
142 hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VDPAU;
143 return *p;
144 break;
145#endif
146#if defined(_WIN32)
147 // Windows pix formats
148 case AV_PIX_FMT_DXVA2_VLD:
149 hw_de_av_pix_fmt_global = AV_PIX_FMT_DXVA2_VLD;
150 hw_de_av_device_type_global = AV_HWDEVICE_TYPE_DXVA2;
151 return *p;
152 break;
153 case AV_PIX_FMT_D3D11:
154 hw_de_av_pix_fmt_global = AV_PIX_FMT_D3D11;
155 hw_de_av_device_type_global = AV_HWDEVICE_TYPE_D3D11VA;
156 return *p;
157 break;
158#endif
159#if defined(__APPLE__)
160 // Apple pix formats
161 case AV_PIX_FMT_VIDEOTOOLBOX:
162 hw_de_av_pix_fmt_global = AV_PIX_FMT_VIDEOTOOLBOX;
163 hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
164 return *p;
165 break;
166#endif
167 // Cross-platform pix formats
168 case AV_PIX_FMT_CUDA:
169 hw_de_av_pix_fmt_global = AV_PIX_FMT_CUDA;
170 hw_de_av_device_type_global = AV_HWDEVICE_TYPE_CUDA;
171 return *p;
172 break;
173 case AV_PIX_FMT_QSV:
174 hw_de_av_pix_fmt_global = AV_PIX_FMT_QSV;
175 hw_de_av_device_type_global = AV_HWDEVICE_TYPE_QSV;
176 return *p;
177 break;
178 default:
179 // This is only here to silence unused-enum warnings
180 break;
181 }
182 }
183 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::get_hw_dec_format (Unable to decode this file using hardware decode)");
184 return AV_PIX_FMT_NONE;
185}
186
187int FFmpegReader::IsHardwareDecodeSupported(int codecid)
188{
189 int ret;
190 switch (codecid) {
191 case AV_CODEC_ID_H264:
192 case AV_CODEC_ID_MPEG2VIDEO:
193 case AV_CODEC_ID_VC1:
194 case AV_CODEC_ID_WMV1:
195 case AV_CODEC_ID_WMV2:
196 case AV_CODEC_ID_WMV3:
197 ret = 1;
198 break;
199 default :
200 ret = 0;
201 break;
202 }
203 return ret;
204}
205#endif // USE_HW_ACCEL
206
208 // Open reader if not already open
209 if (!is_open) {
210 // Prevent async calls to the following code
211 const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
212
213 // Initialize format context
214 pFormatCtx = NULL;
215 {
217 ZmqLogger::Instance()->AppendDebugMethod("Decode hardware acceleration settings", "hw_de_on", hw_de_on, "HARDWARE_DECODER", openshot::Settings::Instance()->HARDWARE_DECODER);
218 }
219
220 // Open video file
221 if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
222 throw InvalidFile("File could not be opened.", path);
223
224 // Retrieve stream information
225 if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
226 throw NoStreamsFound("No streams found in file.", path);
227
228 videoStream = -1;
229 audioStream = -1;
230
231 // Init end-of-file detection variables
232 packet_status.reset(true);
233
234 // Loop through each stream, and identify the video and audio stream index
235 for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
236 // Is this a video stream?
237 if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
238 videoStream = i;
239 packet_status.video_eof = false;
240 packet_status.packets_eof = false;
241 packet_status.end_of_file = false;
242 }
243 // Is this an audio stream?
244 if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
245 audioStream = i;
246 packet_status.audio_eof = false;
247 packet_status.packets_eof = false;
248 packet_status.end_of_file = false;
249 }
250 }
251 if (videoStream == -1 && audioStream == -1)
252 throw NoStreamsFound("No video or audio streams found in this file.", path);
253
254 // Is there a video stream?
255 if (videoStream != -1) {
256 // Set the stream index
257 info.video_stream_index = videoStream;
258
259 // Set the codec and codec context pointers
260 pStream = pFormatCtx->streams[videoStream];
261
262 // Find the codec ID from stream
263 const AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(pStream);
264
265 // Get codec and codec context from stream
266 const AVCodec *pCodec = avcodec_find_decoder(codecId);
267 AVDictionary *opts = NULL;
268 int retry_decode_open = 2;
269 // If hw accel is selected but hardware cannot handle repeat with software decoding
270 do {
271 pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec);
272#if USE_HW_ACCEL
273 if (hw_de_on && (retry_decode_open==2)) {
274 // Up to here no decision is made if hardware or software decode
275 hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id);
276 }
277#endif
278 retry_decode_open = 0;
279
280 // Set number of threads equal to number of processors (not to exceed 16)
281 pCodecCtx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
282
283 if (pCodec == NULL) {
284 throw InvalidCodec("A valid video codec could not be found for this file.", path);
285 }
286
287 // Init options
288 av_dict_set(&opts, "strict", "experimental", 0);
289#if USE_HW_ACCEL
290 if (hw_de_on && hw_de_supported) {
291 // Open Hardware Acceleration
292 int i_decoder_hw = 0;
293 char adapter[256];
294 char *adapter_ptr = NULL;
295 int adapter_num;
297 fprintf(stderr, "Hardware decoding device number: %d\n", adapter_num);
298
299 // Set hardware pix format (callback)
300 pCodecCtx->get_format = get_hw_dec_format;
301
302 if (adapter_num < 3 && adapter_num >=0) {
303#if defined(__unix__)
304 snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128);
305 adapter_ptr = adapter;
307 switch (i_decoder_hw) {
308 case 1:
309 hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
310 break;
311 case 2:
312 hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
313 break;
314 case 6:
315 hw_de_av_device_type = AV_HWDEVICE_TYPE_VDPAU;
316 break;
317 case 7:
318 hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
319 break;
320 default:
321 hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
322 break;
323 }
324
325#elif defined(_WIN32)
326 adapter_ptr = NULL;
328 switch (i_decoder_hw) {
329 case 2:
330 hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
331 break;
332 case 3:
333 hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
334 break;
335 case 4:
336 hw_de_av_device_type = AV_HWDEVICE_TYPE_D3D11VA;
337 break;
338 case 7:
339 hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
340 break;
341 default:
342 hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
343 break;
344 }
345#elif defined(__APPLE__)
346 adapter_ptr = NULL;
348 switch (i_decoder_hw) {
349 case 5:
350 hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
351 break;
352 case 7:
353 hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
354 break;
355 default:
356 hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
357 break;
358 }
359#endif
360
361 } else {
362 adapter_ptr = NULL; // Just to be sure
363 }
364
365 // Check if it is there and writable
366#if defined(__unix__)
367 if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) {
368#elif defined(_WIN32)
369 if( adapter_ptr != NULL ) {
370#elif defined(__APPLE__)
371 if( adapter_ptr != NULL ) {
372#else
373 if( adapter_ptr != NULL ) {
374#endif
375 ZmqLogger::Instance()->AppendDebugMethod("Decode Device present using device");
376 }
377 else {
378 adapter_ptr = NULL; // use default
379 ZmqLogger::Instance()->AppendDebugMethod("Decode Device not present using default");
380 }
381
382 hw_device_ctx = NULL;
383 // Here the first hardware initialisations are made
384 if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) {
385 if (!(pCodecCtx->hw_device_ctx = av_buffer_ref(hw_device_ctx))) {
386 throw InvalidCodec("Hardware device reference create failed.", path);
387 }
388
389 /*
390 av_buffer_unref(&ist->hw_frames_ctx);
391 ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx);
392 if (!ist->hw_frames_ctx) {
393 av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA frames context\n");
394 return AVERROR(ENOMEM);
395 }
396
397 frames_ctx = (AVHWFramesContext*)ist->hw_frames_ctx->data;
398
399 frames_ctx->format = AV_PIX_FMT_CUDA;
400 frames_ctx->sw_format = avctx->sw_pix_fmt;
401 frames_ctx->width = avctx->width;
402 frames_ctx->height = avctx->height;
403
404 av_log(avctx, AV_LOG_DEBUG, "Initializing CUDA frames context: sw_format = %s, width = %d, height = %d\n",
405 av_get_pix_fmt_name(frames_ctx->sw_format), frames_ctx->width, frames_ctx->height);
406
407
408 ret = av_hwframe_ctx_init(pCodecCtx->hw_device_ctx);
409 ret = av_hwframe_ctx_init(ist->hw_frames_ctx);
410 if (ret < 0) {
411 av_log(avctx, AV_LOG_ERROR, "Error initializing a CUDA frame pool\n");
412 return ret;
413 }
414 */
415 }
416 else {
417 throw InvalidCodec("Hardware device create failed.", path);
418 }
419 }
420#endif // USE_HW_ACCEL
421
422 // Disable per-frame threading for album arts
423 // Using FF_THREAD_FRAME adds one frame decoding delay per thread,
424 // but there's only one frame in this case.
425 if (HasAlbumArt())
426 {
427 pCodecCtx->thread_type &= ~FF_THREAD_FRAME;
428 }
429
430 // Open video codec
431 int avcodec_return = avcodec_open2(pCodecCtx, pCodec, &opts);
432 if (avcodec_return < 0) {
433 std::stringstream avcodec_error_msg;
434 avcodec_error_msg << "A video codec was found, but could not be opened. Error: " << av_err2string(avcodec_return);
435 throw InvalidCodec(avcodec_error_msg.str(), path);
436 }
437
438#if USE_HW_ACCEL
439 if (hw_de_on && hw_de_supported) {
440 AVHWFramesConstraints *constraints = NULL;
441 void *hwconfig = NULL;
442 hwconfig = av_hwdevice_hwconfig_alloc(hw_device_ctx);
443
444// TODO: needs va_config!
445#if ENABLE_VAAPI
446 ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config;
447 constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig);
448#endif // ENABLE_VAAPI
449 if (constraints) {
450 if (pCodecCtx->coded_width < constraints->min_width ||
451 pCodecCtx->coded_height < constraints->min_height ||
452 pCodecCtx->coded_width > constraints->max_width ||
453 pCodecCtx->coded_height > constraints->max_height) {
454 ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n");
455 hw_de_supported = 0;
456 retry_decode_open = 1;
457 AV_FREE_CONTEXT(pCodecCtx);
458 if (hw_device_ctx) {
459 av_buffer_unref(&hw_device_ctx);
460 hw_device_ctx = NULL;
461 }
462 }
463 else {
464 // All is just peachy
465 ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Min width :", constraints->min_width, "Min Height :", constraints->min_height, "MaxWidth :", constraints->max_width, "MaxHeight :", constraints->max_height, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
466 retry_decode_open = 0;
467 }
468 av_hwframe_constraints_free(&constraints);
469 if (hwconfig) {
470 av_freep(&hwconfig);
471 }
472 }
473 else {
474 int max_h, max_w;
475 //max_h = ((getenv( "LIMIT_HEIGHT_MAX" )==NULL) ? MAX_SUPPORTED_HEIGHT : atoi(getenv( "LIMIT_HEIGHT_MAX" )));
477 //max_w = ((getenv( "LIMIT_WIDTH_MAX" )==NULL) ? MAX_SUPPORTED_WIDTH : atoi(getenv( "LIMIT_WIDTH_MAX" )));
479 ZmqLogger::Instance()->AppendDebugMethod("Constraints could not be found using default limit\n");
480 //cerr << "Constraints could not be found using default limit\n";
481 if (pCodecCtx->coded_width < 0 ||
482 pCodecCtx->coded_height < 0 ||
483 pCodecCtx->coded_width > max_w ||
484 pCodecCtx->coded_height > max_h ) {
485 ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
486 hw_de_supported = 0;
487 retry_decode_open = 1;
488 AV_FREE_CONTEXT(pCodecCtx);
489 if (hw_device_ctx) {
490 av_buffer_unref(&hw_device_ctx);
491 hw_device_ctx = NULL;
492 }
493 }
494 else {
495 ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
496 retry_decode_open = 0;
497 }
498 }
499 } // if hw_de_on && hw_de_supported
500 else {
501 ZmqLogger::Instance()->AppendDebugMethod("\nDecode in software is used\n");
502 }
503#else
504 retry_decode_open = 0;
505#endif // USE_HW_ACCEL
506 } while (retry_decode_open); // retry_decode_open
507 // Free options
508 av_dict_free(&opts);
509
510 // Update the File Info struct with video details (if a video stream is found)
511 UpdateVideoInfo();
512 }
513
514 // Is there an audio stream?
515 if (audioStream != -1) {
516 // Set the stream index
517 info.audio_stream_index = audioStream;
518
519 // Get a pointer to the codec context for the audio stream
520 aStream = pFormatCtx->streams[audioStream];
521
522 // Find the codec ID from stream
523 AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(aStream);
524
525 // Get codec and codec context from stream
526 const AVCodec *aCodec = avcodec_find_decoder(codecId);
527 aCodecCtx = AV_GET_CODEC_CONTEXT(aStream, aCodec);
528
529 // Set number of threads equal to number of processors (not to exceed 16)
530 aCodecCtx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
531
532 if (aCodec == NULL) {
533 throw InvalidCodec("A valid audio codec could not be found for this file.", path);
534 }
535
536 // Init options
537 AVDictionary *opts = NULL;
538 av_dict_set(&opts, "strict", "experimental", 0);
539
540 // Open audio codec
541 if (avcodec_open2(aCodecCtx, aCodec, &opts) < 0)
542 throw InvalidCodec("An audio codec was found, but could not be opened.", path);
543
544 // Free options
545 av_dict_free(&opts);
546
547 // Update the File Info struct with audio details (if an audio stream is found)
548 UpdateAudioInfo();
549 }
550
551 // Add format metadata (if any)
552 AVDictionaryEntry *tag = NULL;
553 while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
554 QString str_key = tag->key;
555 QString str_value = tag->value;
556 info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
557 }
558
559 // Init previous audio location to zero
560 previous_packet_location.frame = -1;
561 previous_packet_location.sample_start = 0;
562
563 // Adjust cache size based on size of frame and audio
564 working_cache.SetMaxBytesFromInfo(max_concurrent_frames * info.fps.ToDouble() * 2, info.width, info.height, info.sample_rate, info.channels);
566
567 // Scan PTS for any offsets (i.e. non-zero starting streams). At least 1 stream must start at zero timestamp.
568 // This method allows us to shift timestamps to ensure at least 1 stream is starting at zero.
569 UpdatePTSOffset();
570
571 // Override an invalid framerate
572 if (info.fps.ToFloat() > 240.0f || (info.fps.num <= 0 || info.fps.den <= 0) || info.video_length <= 0) {
573 // Calculate FPS, duration, video bit rate, and video length manually
574 // by scanning through all the video stream packets
575 CheckFPS();
576 }
577
578 // Mark as "open"
579 is_open = true;
580
581 // Seek back to beginning of file (if not already seeking)
582 if (!is_seeking) {
583 Seek(1);
584 }
585 }
586}
587
589 // Close all objects, if reader is 'open'
590 if (is_open) {
591 // Prevent async calls to the following code
592 const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
593
594 // Mark as "closed"
595 is_open = false;
596
597 // Keep track of most recent packet
598 AVPacket *recent_packet = packet;
599
600 // Drain any packets from the decoder
601 packet = NULL;
602 int attempts = 0;
603 int max_attempts = 128;
604 while (packet_status.packets_decoded() < packet_status.packets_read() && attempts < max_attempts) {
605 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close (Drain decoder loop)",
606 "packets_read", packet_status.packets_read(),
607 "packets_decoded", packet_status.packets_decoded(),
608 "attempts", attempts);
609 if (packet_status.video_decoded < packet_status.video_read) {
610 ProcessVideoPacket(info.video_length);
611 }
612 if (packet_status.audio_decoded < packet_status.audio_read) {
613 ProcessAudioPacket(info.video_length);
614 }
615 attempts++;
616 }
617
618 // Remove packet
619 if (recent_packet) {
620 RemoveAVPacket(recent_packet);
621 }
622
623 // Close the video codec
624 if (info.has_video) {
625 if(avcodec_is_open(pCodecCtx)) {
626 avcodec_flush_buffers(pCodecCtx);
627 }
628 AV_FREE_CONTEXT(pCodecCtx);
629#if USE_HW_ACCEL
630 if (hw_de_on) {
631 if (hw_device_ctx) {
632 av_buffer_unref(&hw_device_ctx);
633 hw_device_ctx = NULL;
634 }
635 }
636#endif // USE_HW_ACCEL
637 }
638
639 // Close the audio codec
640 if (info.has_audio) {
641 if(avcodec_is_open(aCodecCtx)) {
642 avcodec_flush_buffers(aCodecCtx);
643 }
644 AV_FREE_CONTEXT(aCodecCtx);
645 }
646
647 // Clear final cache
649 working_cache.Clear();
650
651 // Close the video file
652 avformat_close_input(&pFormatCtx);
653 av_freep(&pFormatCtx);
654
655 // Reset some variables
656 last_frame = 0;
657 hold_packet = false;
658 largest_frame_processed = 0;
659 seek_audio_frame_found = 0;
660 seek_video_frame_found = 0;
661 current_video_frame = 0;
662 last_video_frame.reset();
663 }
664}
665
666bool FFmpegReader::HasAlbumArt() {
667 // Check if the video stream we use is an attached picture
668 // This won't return true if the file has a cover image as a secondary stream
669 // like an MKV file with an attached image file
670 return pFormatCtx && videoStream >= 0 && pFormatCtx->streams[videoStream]
671 && (pFormatCtx->streams[videoStream]->disposition & AV_DISPOSITION_ATTACHED_PIC);
672}
673
674void FFmpegReader::UpdateAudioInfo() {
675 // Set default audio channel layout (if needed)
676 if (AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
677 AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels);
678
679 if (info.sample_rate > 0) {
680 // Skip init - if info struct already populated
681 return;
682 }
683
684 // Set values of FileInfo struct
685 info.has_audio = true;
686 info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
687 info.acodec = aCodecCtx->codec->name;
688 info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
689 info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout;
690 info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate;
691 info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate;
692 if (info.audio_bit_rate <= 0) {
693 // Get bitrate from format
694 info.audio_bit_rate = pFormatCtx->bit_rate;
695 }
696
697 // Set audio timebase
698 info.audio_timebase.num = aStream->time_base.num;
699 info.audio_timebase.den = aStream->time_base.den;
700
701 // Get timebase of audio stream (if valid) and greater than the current duration
702 if (aStream->duration > 0 && aStream->duration > info.duration) {
703 // Get duration from audio stream
704 info.duration = aStream->duration * info.audio_timebase.ToDouble();
705 } else if (pFormatCtx->duration > 0 && info.duration <= 0.0f) {
706 // Use the format's duration
707 info.duration = float(pFormatCtx->duration) / AV_TIME_BASE;
708 }
709
710 // Calculate duration from filesize and bitrate (if any)
711 if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0) {
712 // Estimate from bitrate, total bytes, and framerate
714 }
715
716 // Check for an invalid video length
717 if (info.has_video && info.video_length <= 0) {
718 // Calculate the video length from the audio duration
720 }
721
722 // Set video timebase (if no video stream was found)
723 if (!info.has_video) {
724 // Set a few important default video settings (so audio can be divided into frames)
725 info.fps.num = 24;
726 info.fps.den = 1;
730 info.width = 720;
731 info.height = 480;
732
733 // Use timeline to set correct width & height (if any)
734 Clip *parent = static_cast<Clip *>(ParentClip());
735 if (parent) {
736 if (parent->ParentTimeline()) {
737 // Set max width/height based on parent clip's timeline (if attached to a timeline)
740 }
741 }
742 }
743
744 // Fix invalid video lengths for certain types of files (MP3 for example)
745 if (info.has_video && ((info.duration * info.fps.ToDouble()) - info.video_length > 60)) {
747 }
748
749 // Add audio metadata (if any found)
750 AVDictionaryEntry *tag = NULL;
751 while ((tag = av_dict_get(aStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
752 QString str_key = tag->key;
753 QString str_value = tag->value;
754 info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
755 }
756}
757
758void FFmpegReader::UpdateVideoInfo() {
759 if (info.vcodec.length() > 0) {
760 // Skip init - if info struct already populated
761 return;
762 }
763
764 // Set values of FileInfo struct
765 info.has_video = true;
766 info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
767 info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
768 info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
769 info.vcodec = pCodecCtx->codec->name;
770 info.video_bit_rate = (pFormatCtx->bit_rate / 8);
771
772 // Frame rate from the container and codec
773 AVRational framerate = av_guess_frame_rate(pFormatCtx, pStream, NULL);
774 if (!check_fps) {
775 info.fps.num = framerate.num;
776 info.fps.den = framerate.den;
777 }
778
779 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo", "info.fps.num", info.fps.num, "info.fps.den", info.fps.den);
780
781 // TODO: remove excessive debug info in the next releases
782 // The debug info below is just for comparison and troubleshooting on users side during the transition period
783 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo (pStream->avg_frame_rate)", "num", pStream->avg_frame_rate.num, "den", pStream->avg_frame_rate.den);
784
785 if (pStream->sample_aspect_ratio.num != 0) {
786 info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
787 info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
788 } else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0) {
789 info.pixel_ratio.num = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num;
790 info.pixel_ratio.den = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.den;
791 } else {
792 info.pixel_ratio.num = 1;
793 info.pixel_ratio.den = 1;
794 }
795 info.pixel_format = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
796
797 // Calculate the DAR (display aspect ratio)
799
800 // Reduce size fraction
801 size.Reduce();
802
803 // Set the ratio based on the reduced fraction
804 info.display_ratio.num = size.num;
805 info.display_ratio.den = size.den;
806
807 // Get scan type and order from codec context/params
808 if (!check_interlace) {
809 check_interlace = true;
810 AVFieldOrder field_order = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->field_order;
811 switch(field_order) {
812 case AV_FIELD_PROGRESSIVE:
813 info.interlaced_frame = false;
814 break;
815 case AV_FIELD_TT:
816 case AV_FIELD_TB:
817 info.interlaced_frame = true;
818 info.top_field_first = true;
819 break;
820 case AV_FIELD_BT:
821 case AV_FIELD_BB:
822 info.interlaced_frame = true;
823 info.top_field_first = false;
824 break;
825 case AV_FIELD_UNKNOWN:
826 // Check again later?
827 check_interlace = false;
828 break;
829 }
830 // check_interlace will prevent these checks being repeated,
831 // unless it was cleared because we got an AV_FIELD_UNKNOWN response.
832 }
833
834 // Set the video timebase
835 info.video_timebase.num = pStream->time_base.num;
836 info.video_timebase.den = pStream->time_base.den;
837
838 // Set the duration in seconds, and video length (# of frames)
839 info.duration = pStream->duration * info.video_timebase.ToDouble();
840
841 // Check for valid duration (if found)
842 if (info.duration <= 0.0f && pFormatCtx->duration >= 0) {
843 // Use the format's duration
844 info.duration = float(pFormatCtx->duration) / AV_TIME_BASE;
845 }
846
847 // Calculate duration from filesize and bitrate (if any)
848 if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0) {
849 // Estimate from bitrate, total bytes, and framerate
851 }
852
853 // Certain "image" formats do not have a valid duration
854 if (info.duration <= 0.0f && pStream->duration == AV_NOPTS_VALUE && pFormatCtx->duration == AV_NOPTS_VALUE) {
855 // Force an "image" duration
856 info.duration = 60 * 60 * 1; // 1 hour duration
857 info.video_length = 1;
858 info.has_single_image = true;
859 }
860
861 // Get the # of video frames (if found in stream)
862 // Only set this 1 time (this method can be called multiple times)
863 if (pStream->nb_frames > 0 && info.video_length <= 0) {
864 info.video_length = pStream->nb_frames;
865 }
866
867 // No duration found in stream of file
868 if (info.duration <= 0.0f) {
869 // No duration is found in the video stream
870 info.duration = -1;
871 info.video_length = -1;
872 is_duration_known = false;
873 } else {
874 // Yes, a duration was found
875 is_duration_known = true;
876
877 // Calculate number of frames (if not already found in metadata)
878 // Only set this 1 time (this method can be called multiple times)
879 if (info.video_length <= 0) {
881 }
882 }
883
884 // Add video metadata (if any)
885 AVDictionaryEntry *tag = NULL;
886 while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
887 QString str_key = tag->key;
888 QString str_value = tag->value;
889 info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
890 }
891}
892
894 return this->is_duration_known;
895}
896
897std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame) {
898 // Check for open reader (or throw exception)
899 if (!is_open)
900 throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
901
902 // Adjust for a requested frame that is too small or too large
903 if (requested_frame < 1)
904 requested_frame = 1;
905 if (requested_frame > info.video_length && is_duration_known)
906 requested_frame = info.video_length;
907 if (info.has_video && info.video_length == 0)
908 // Invalid duration of video file
909 throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
910
911 // Debug output
912 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame);
913
914 // Check the cache for this frame
915 std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
916 if (frame) {
917 // Debug output
918 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame);
919
920 // Return the cached frame
921 return frame;
922 } else {
923
924 // Prevent async calls to the remainder of this code
925 const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
926
927 // Check the cache a 2nd time (due to the potential previous lock)
928 frame = final_cache.GetFrame(requested_frame);
929 if (frame) {
930 // Debug output
931 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame);
932
933 } else {
934 // Frame is not in cache
935 // Reset seek count
936 seek_count = 0;
937
938 // Are we within X frames of the requested frame?
939 int64_t diff = requested_frame - last_frame;
940 if (diff >= 1 && diff <= 20) {
941 // Continue walking the stream
942 frame = ReadStream(requested_frame);
943 } else {
944 // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
945 if (enable_seek) {
946 // Only seek if enabled
947 Seek(requested_frame);
948
949 } else if (!enable_seek && diff < 0) {
950 // Start over, since we can't seek, and the requested frame is smaller than our position
951 // Since we are seeking to frame 1, this actually just closes/re-opens the reader
952 Seek(1);
953 }
954
955 // Then continue walking the stream
956 frame = ReadStream(requested_frame);
957 }
958 }
959 return frame;
960 }
961}
962
963// Read the stream until we find the requested Frame
964std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame) {
965 // Allocate video frame
966 bool check_seek = false;
967 int packet_error = -1;
968
969 // Debug output
970 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame, "max_concurrent_frames", max_concurrent_frames);
971
972 // Loop through the stream until the correct frame is found
973 while (true) {
974 // Check if working frames are 'finished'
975 if (!is_seeking) {
976 // Check for final frames
977 CheckWorkingFrames(requested_frame);
978 }
979
980 // Check if requested 'final' frame is available (and break out of loop if found)
981 bool is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
982 if (is_cache_found) {
983 break;
984 }
985
986 if (!hold_packet || !packet) {
987 // Get the next packet
988 packet_error = GetNextPacket();
989 if (packet_error < 0 && !packet) {
990 // No more packets to be found
991 packet_status.packets_eof = true;
992 }
993 }
994
995 // Debug output
996 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame,"packets_read", packet_status.packets_read(), "packets_decoded", packet_status.packets_decoded(), "is_seeking", is_seeking);
997
998 // Check the status of a seek (if any)
999 if (is_seeking) {
1000 check_seek = CheckSeek(false);
1001 } else {
1002 check_seek = false;
1003 }
1004
1005 if (check_seek) {
1006 // Packet may become NULL on Close inside Seek if CheckSeek returns false
1007 // Jump to the next iteration of this loop
1008 continue;
1009 }
1010
1011 // Video packet
1012 if ((info.has_video && packet && packet->stream_index == videoStream) ||
1013 (info.has_video && packet_status.video_decoded < packet_status.video_read) ||
1014 (info.has_video && !packet && !packet_status.video_eof)) {
1015 // Process Video Packet
1016 ProcessVideoPacket(requested_frame);
1017 }
1018 // Audio packet
1019 if ((info.has_audio && packet && packet->stream_index == audioStream) ||
1020 (info.has_audio && !packet && packet_status.audio_decoded < packet_status.audio_read) ||
1021 (info.has_audio && !packet && !packet_status.audio_eof)) {
1022 // Process Audio Packet
1023 ProcessAudioPacket(requested_frame);
1024 }
1025
1026 // Remove unused packets (sometimes we purposely ignore video or audio packets,
1027 // if the has_video or has_audio properties are manually overridden)
1028 if ((!info.has_video && packet && packet->stream_index == videoStream) ||
1029 (!info.has_audio && packet && packet->stream_index == audioStream)) {
1030 // Keep track of deleted packet counts
1031 if (packet->stream_index == videoStream) {
1032 packet_status.video_decoded++;
1033 } else if (packet->stream_index == audioStream) {
1034 packet_status.audio_decoded++;
1035 }
1036
1037 // Remove unused packets (sometimes we purposely ignore video or audio packets,
1038 // if the has_video or has_audio properties are manually overridden)
1039 RemoveAVPacket(packet);
1040 packet = NULL;
1041 }
1042
1043 // Determine end-of-stream (waiting until final decoder threads finish)
1044 // Force end-of-stream in some situations
1045 packet_status.end_of_file = packet_status.packets_eof && packet_status.video_eof && packet_status.audio_eof;
1046 if ((packet_status.packets_eof && packet_status.packets_read() == packet_status.packets_decoded()) || packet_status.end_of_file) {
1047 // Force EOF (end of file) variables to true, if decoder does not support EOF detection.
1048 // If we have no more packets, and all known packets have been decoded
1049 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (force EOF)", "packets_read", packet_status.packets_read(), "packets_decoded", packet_status.packets_decoded(), "packets_eof", packet_status.packets_eof, "video_eof", packet_status.video_eof, "audio_eof", packet_status.audio_eof, "end_of_file", packet_status.end_of_file);
1050 if (!packet_status.video_eof) {
1051 packet_status.video_eof = true;
1052 }
1053 if (!packet_status.audio_eof) {
1054 packet_status.audio_eof = true;
1055 }
1056 packet_status.end_of_file = true;
1057 break;
1058 }
1059 } // end while
1060
1061 // Debug output
1062 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)",
1063 "packets_read", packet_status.packets_read(),
1064 "packets_decoded", packet_status.packets_decoded(),
1065 "end_of_file", packet_status.end_of_file,
1066 "largest_frame_processed", largest_frame_processed,
1067 "Working Cache Count", working_cache.Count());
1068
1069 // Have we reached end-of-stream (or the final frame)?
1070 if (!packet_status.end_of_file && requested_frame >= info.video_length) {
1071 // Force end-of-stream
1072 packet_status.end_of_file = true;
1073 }
1074 if (packet_status.end_of_file) {
1075 // Mark any other working frames as 'finished'
1076 CheckWorkingFrames(requested_frame);
1077 }
1078
1079 // Return requested frame (if found)
1080 std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
1081 if (frame)
1082 // Return prepared frame
1083 return frame;
1084 else {
1085
1086 // Check if largest frame is still cached
1087 frame = final_cache.GetFrame(largest_frame_processed);
1088 int samples_in_frame = Frame::GetSamplesPerFrame(requested_frame, info.fps,
1090 if (frame) {
1091 // Copy and return the largest processed frame (assuming it was the last in the video file)
1092 std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1093
1094 // Use solid color (if no image data found)
1095 if (!frame->has_image_data) {
1096 // Use solid black frame if no image data available
1097 f->AddColor(info.width, info.height, "#000");
1098 }
1099 // Silence audio data (if any), since we are repeating the last frame
1100 frame->AddAudioSilence(samples_in_frame);
1101
1102 return frame;
1103 } else {
1104 // The largest processed frame is no longer in cache, return a blank frame
1105 std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1106 f->AddColor(info.width, info.height, "#000");
1107 f->AddAudioSilence(samples_in_frame);
1108 return f;
1109 }
1110 }
1111
1112}
1113
1114// Get the next packet (if any)
1115int FFmpegReader::GetNextPacket() {
1116 int found_packet = 0;
1117 AVPacket *next_packet;
1118 next_packet = new AVPacket();
1119 found_packet = av_read_frame(pFormatCtx, next_packet);
1120
1121 if (packet) {
1122 // Remove previous packet before getting next one
1123 RemoveAVPacket(packet);
1124 packet = NULL;
1125 }
1126 if (found_packet >= 0) {
1127 // Update current packet pointer
1128 packet = next_packet;
1129
1130 // Keep track of packet stats
1131 if (packet->stream_index == videoStream) {
1132 packet_status.video_read++;
1133 } else if (packet->stream_index == audioStream) {
1134 packet_status.audio_read++;
1135 }
1136 } else {
1137 // No more packets found
1138 delete next_packet;
1139 packet = NULL;
1140 }
1141 // Return if packet was found (or error number)
1142 return found_packet;
1143}
1144
1145// Get an AVFrame (if any)
1146bool FFmpegReader::GetAVFrame() {
1147 int frameFinished = 0;
1148
1149 // Decode video frame
1150 AVFrame *next_frame = AV_ALLOCATE_FRAME();
1151
1152#if IS_FFMPEG_3_2
1153 int send_packet_err = 0;
1154 int64_t send_packet_pts = 0;
1155 if ((packet && packet->stream_index == videoStream) || !packet) {
1156 send_packet_err = avcodec_send_packet(pCodecCtx, packet);
1157
1158 if (packet && send_packet_err >= 0) {
1159 send_packet_pts = GetPacketPTS();
1160 hold_packet = false;
1161 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet succeeded)", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts);
1162 }
1163 }
1164
1165 #if USE_HW_ACCEL
1166 // Get the format from the variables set in get_hw_dec_format
1167 hw_de_av_pix_fmt = hw_de_av_pix_fmt_global;
1168 hw_de_av_device_type = hw_de_av_device_type_global;
1169 #endif // USE_HW_ACCEL
1170 if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) {
1171 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: Not sent [" + av_err2string(send_packet_err) + "])", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts);
1172 if (send_packet_err == AVERROR(EAGAIN)) {
1173 hold_packet = true;
1174 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(EAGAIN): user must read output with avcodec_receive_frame()", "send_packet_pts", send_packet_pts);
1175 }
1176 if (send_packet_err == AVERROR(EINVAL)) {
1177 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(EINVAL): codec not opened, it is an encoder, or requires flush", "send_packet_pts", send_packet_pts);
1178 }
1179 if (send_packet_err == AVERROR(ENOMEM)) {
1180 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(ENOMEM): failed to add packet to internal queue, or legitimate decoding errors", "send_packet_pts", send_packet_pts);
1181 }
1182 }
1183
1184 // Always try and receive a packet, if not EOF.
1185 // Even if the above avcodec_send_packet failed to send,
1186 // we might still need to receive a packet.
1187 int receive_frame_err = 0;
1188 AVFrame *next_frame2;
1189#if USE_HW_ACCEL
1190 if (hw_de_on && hw_de_supported) {
1191 next_frame2 = AV_ALLOCATE_FRAME();
1192 }
1193 else
1194#endif // USE_HW_ACCEL
1195 {
1196 next_frame2 = next_frame;
1197 }
1198 pFrame = AV_ALLOCATE_FRAME();
1199 while (receive_frame_err >= 0) {
1200 receive_frame_err = avcodec_receive_frame(pCodecCtx, next_frame2);
1201
1202 if (receive_frame_err != 0) {
1203 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (receive frame: frame not ready yet from decoder [\" + av_err2string(receive_frame_err) + \"])", "receive_frame_err", receive_frame_err, "send_packet_pts", send_packet_pts);
1204
1205 if (receive_frame_err == AVERROR_EOF) {
1207 "FFmpegReader::GetAVFrame (receive frame: AVERROR_EOF: EOF detected from decoder, flushing buffers)", "send_packet_pts", send_packet_pts);
1208 avcodec_flush_buffers(pCodecCtx);
1209 packet_status.video_eof = true;
1210 }
1211 if (receive_frame_err == AVERROR(EINVAL)) {
1213 "FFmpegReader::GetAVFrame (receive frame: AVERROR(EINVAL): invalid frame received, flushing buffers)", "send_packet_pts", send_packet_pts);
1214 avcodec_flush_buffers(pCodecCtx);
1215 }
1216 if (receive_frame_err == AVERROR(EAGAIN)) {
1218 "FFmpegReader::GetAVFrame (receive frame: AVERROR(EAGAIN): output is not available in this state - user must try to send new input)", "send_packet_pts", send_packet_pts);
1219 }
1220 if (receive_frame_err == AVERROR_INPUT_CHANGED) {
1222 "FFmpegReader::GetAVFrame (receive frame: AVERROR_INPUT_CHANGED: current decoded frame has changed parameters with respect to first decoded frame)", "send_packet_pts", send_packet_pts);
1223 }
1224
1225 // Break out of decoding loop
1226 // Nothing ready for decoding yet
1227 break;
1228 }
1229
1230#if USE_HW_ACCEL
1231 if (hw_de_on && hw_de_supported) {
1232 int err;
1233 if (next_frame2->format == hw_de_av_pix_fmt) {
1234 next_frame->format = AV_PIX_FMT_YUV420P;
1235 if ((err = av_hwframe_transfer_data(next_frame,next_frame2,0)) < 0) {
1236 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to transfer data to output frame)", "hw_de_on", hw_de_on);
1237 }
1238 if ((err = av_frame_copy_props(next_frame,next_frame2)) < 0) {
1239 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to copy props to output frame)", "hw_de_on", hw_de_on);
1240 }
1241 }
1242 }
1243 else
1244#endif // USE_HW_ACCEL
1245 { // No hardware acceleration used -> no copy from GPU memory needed
1246 next_frame = next_frame2;
1247 }
1248
1249 // TODO also handle possible further frames
1250 // Use only the first frame like avcodec_decode_video2
1251 frameFinished = 1;
1252 packet_status.video_decoded++;
1253
1254 av_image_alloc(pFrame->data, pFrame->linesize, info.width, info.height, (AVPixelFormat)(pStream->codecpar->format), 1);
1255 av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)next_frame->data, next_frame->linesize,
1256 (AVPixelFormat)(pStream->codecpar->format), info.width, info.height);
1257
1258 // Get display PTS from video frame, often different than packet->pts.
1259 // Sending packets to the decoder (i.e. packet->pts) is async,
1260 // and retrieving packets from the decoder (frame->pts) is async. In most decoders
1261 // sending and retrieving are separated by multiple calls to this method.
1262 if (next_frame->pts != AV_NOPTS_VALUE) {
1263 // This is the current decoded frame (and should be the pts used) for
1264 // processing this data
1265 video_pts = next_frame->pts;
1266 } else if (next_frame->pkt_dts != AV_NOPTS_VALUE) {
1267 // Some videos only set this timestamp (fallback)
1268 video_pts = next_frame->pkt_dts;
1269 }
1270
1272 "FFmpegReader::GetAVFrame (Successful frame received)", "video_pts", video_pts, "send_packet_pts", send_packet_pts);
1273
1274 // break out of loop after each successful image returned
1275 break;
1276 }
1277#if USE_HW_ACCEL
1278 if (hw_de_on && hw_de_supported) {
1279 AV_FREE_FRAME(&next_frame2);
1280 }
1281 #endif // USE_HW_ACCEL
1282#else
1283 avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
1284
1285 // always allocate pFrame (because we do that in the ffmpeg >= 3.2 as well); it will always be freed later
1286 pFrame = AV_ALLOCATE_FRAME();
1287
1288 // is frame finished
1289 if (frameFinished) {
1290 // AVFrames are clobbered on the each call to avcodec_decode_video, so we
1291 // must make a copy of the image data before this method is called again.
1292 avpicture_alloc((AVPicture *) pFrame, pCodecCtx->pix_fmt, info.width, info.height);
1293 av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width,
1294 info.height);
1295 }
1296#endif // IS_FFMPEG_3_2
1297
1298 // deallocate the frame
1299 AV_FREE_FRAME(&next_frame);
1300
1301 // Did we get a video frame?
1302 return frameFinished;
1303}
1304
1305// Check the current seek position and determine if we need to seek again
1306bool FFmpegReader::CheckSeek(bool is_video) {
1307 // Are we seeking for a specific frame?
1308 if (is_seeking) {
1309 // Determine if both an audio and video packet have been decoded since the seek happened.
1310 // If not, allow the ReadStream method to keep looping
1311 if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
1312 return false;
1313
1314 // Check for both streams
1315 if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
1316 return false;
1317
1318 // Determine max seeked frame
1319 int64_t max_seeked_frame = std::max(seek_audio_frame_found, seek_video_frame_found);
1320
1321 // determine if we are "before" the requested frame
1322 if (max_seeked_frame >= seeking_frame) {
1323 // SEEKED TOO FAR
1324 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)",
1325 "is_video_seek", is_video_seek,
1326 "max_seeked_frame", max_seeked_frame,
1327 "seeking_frame", seeking_frame,
1328 "seeking_pts", seeking_pts,
1329 "seek_video_frame_found", seek_video_frame_found,
1330 "seek_audio_frame_found", seek_audio_frame_found);
1331
1332 // Seek again... to the nearest Keyframe
1333 Seek(seeking_frame - (10 * seek_count * seek_count));
1334 } else {
1335 // SEEK WORKED
1336 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)",
1337 "is_video_seek", is_video_seek,
1338 "packet->pts", GetPacketPTS(),
1339 "seeking_pts", seeking_pts,
1340 "seeking_frame", seeking_frame,
1341 "seek_video_frame_found", seek_video_frame_found,
1342 "seek_audio_frame_found", seek_audio_frame_found);
1343
1344 // Seek worked, and we are "before" the requested frame
1345 is_seeking = false;
1346 seeking_frame = 0;
1347 seeking_pts = -1;
1348 }
1349 }
1350
1351 // return the pts to seek to (if any)
1352 return is_seeking;
1353}
1354
1355// Process a video packet
1356void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
1357 // Get the AVFrame from the current packet
1358 // This sets the video_pts to the correct timestamp
1359 int frame_finished = GetAVFrame();
1360
1361 // Check if the AVFrame is finished and set it
1362 if (!frame_finished) {
1363 // No AVFrame decoded yet, bail out
1364 if (pFrame) {
1365 RemoveAVFrame(pFrame);
1366 }
1367 return;
1368 }
1369
1370 // Calculate current frame #
1371 int64_t current_frame = ConvertVideoPTStoFrame(video_pts);
1372
1373 // Track 1st video packet after a successful seek
1374 if (!seek_video_frame_found && is_seeking)
1375 seek_video_frame_found = current_frame;
1376
1377 // Create or get the existing frame object. Requested frame needs to be created
1378 // in working_cache at least once. Seek can clear the working_cache, so we must
1379 // add the requested frame back to the working_cache here. If it already exists,
1380 // it will be moved to the top of the working_cache.
1381 working_cache.Add(CreateFrame(requested_frame));
1382
1383 // Debug output
1384 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame);
1385
1386 // Init some things local (for OpenMP)
1387 PixelFormat pix_fmt = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
1388 int height = info.height;
1389 int width = info.width;
1390 int64_t video_length = info.video_length;
1391
1392 // Create variables for a RGB Frame (since most videos are not in RGB, we must convert it)
1393 AVFrame *pFrameRGB = nullptr;
1394 uint8_t *buffer = nullptr;
1395
1396 // Allocate an AVFrame structure
1397 pFrameRGB = AV_ALLOCATE_FRAME();
1398 if (pFrameRGB == nullptr)
1399 throw OutOfMemory("Failed to allocate frame buffer", path);
1400
1401 // Determine the max size of this source image (based on the timeline's size, the scaling mode,
1402 // and the scaling keyframes). This is a performance improvement, to keep the images as small as possible,
1403 // without losing quality. NOTE: We cannot go smaller than the timeline itself, or the add_layer timeline
1404 // method will scale it back to timeline size before scaling it smaller again. This needs to be fixed in
1405 // the future.
1406 int max_width = info.width;
1407 int max_height = info.height;
1408
1409 Clip *parent = static_cast<Clip *>(ParentClip());
1410 if (parent) {
1411 if (parent->ParentTimeline()) {
1412 // Set max width/height based on parent clip's timeline (if attached to a timeline)
1413 max_width = parent->ParentTimeline()->preview_width;
1414 max_height = parent->ParentTimeline()->preview_height;
1415 }
1416 if (parent->scale == SCALE_FIT || parent->scale == SCALE_STRETCH) {
1417 // Best fit or Stretch scaling (based on max timeline size * scaling keyframes)
1418 float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1419 float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1420 max_width = std::max(float(max_width), max_width * max_scale_x);
1421 max_height = std::max(float(max_height), max_height * max_scale_y);
1422
1423 } else if (parent->scale == SCALE_CROP) {
1424 // Cropping scale mode (based on max timeline size * cropped size * scaling keyframes)
1425 float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1426 float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1427 QSize width_size(max_width * max_scale_x,
1428 round(max_width / (float(info.width) / float(info.height))));
1429 QSize height_size(round(max_height / (float(info.height) / float(info.width))),
1430 max_height * max_scale_y);
1431 // respect aspect ratio
1432 if (width_size.width() >= max_width && width_size.height() >= max_height) {
1433 max_width = std::max(max_width, width_size.width());
1434 max_height = std::max(max_height, width_size.height());
1435 } else {
1436 max_width = std::max(max_width, height_size.width());
1437 max_height = std::max(max_height, height_size.height());
1438 }
1439
1440 } else {
1441 // Scale video to equivalent unscaled size
1442 // Since the preview window can change sizes, we want to always
1443 // scale against the ratio of original video size to timeline size
1444 float preview_ratio = 1.0;
1445 if (parent->ParentTimeline()) {
1446 Timeline *t = (Timeline *) parent->ParentTimeline();
1447 preview_ratio = t->preview_width / float(t->info.width);
1448 }
1449 float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1450 float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1451 max_width = info.width * max_scale_x * preview_ratio;
1452 max_height = info.height * max_scale_y * preview_ratio;
1453 }
1454 }
1455
1456 // Determine if image needs to be scaled (for performance reasons)
1457 int original_height = height;
1458 if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
1459 // Override width and height (but maintain aspect ratio)
1460 float ratio = float(width) / float(height);
1461 int possible_width = round(max_height * ratio);
1462 int possible_height = round(max_width / ratio);
1463
1464 if (possible_width <= max_width) {
1465 // use calculated width, and max_height
1466 width = possible_width;
1467 height = max_height;
1468 } else {
1469 // use max_width, and calculated height
1470 width = max_width;
1471 height = possible_height;
1472 }
1473 }
1474
1475 // Determine required buffer size and allocate buffer
1476 const int bytes_per_pixel = 4;
1477 int buffer_size = (width * height * bytes_per_pixel) + 128;
1478 buffer = new unsigned char[buffer_size]();
1479
1480 // Copy picture data from one AVFrame (or AVPicture) to another one.
1481 AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
1482
1483 int scale_mode = SWS_FAST_BILINEAR;
1484 if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
1485 scale_mode = SWS_BICUBIC;
1486 }
1487 SwsContext *img_convert_ctx = sws_getContext(info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx), width,
1488 height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL);
1489
1490 // Resize / Convert to RGB
1491 sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0,
1492 original_height, pFrameRGB->data, pFrameRGB->linesize);
1493
1494 // Create or get the existing frame object
1495 std::shared_ptr<Frame> f = CreateFrame(current_frame);
1496
1497 // Add Image data to frame
1498 if (!ffmpeg_has_alpha(AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx))) {
1499 // Add image with no alpha channel, Speed optimization
1500 f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888_Premultiplied, buffer);
1501 } else {
1502 // Add image with alpha channel (this will be converted to premultipled when needed, but is slower)
1503 f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888, buffer);
1504 }
1505
1506 // Update working cache
1507 working_cache.Add(f);
1508
1509 // Keep track of last last_video_frame
1510 last_video_frame = f;
1511
1512 // Free the RGB image
1513 AV_FREE_FRAME(&pFrameRGB);
1514
1515 // Remove frame and packet
1516 RemoveAVFrame(pFrame);
1517 sws_freeContext(img_convert_ctx);
1518
1519 // Get video PTS in seconds
1520 video_pts_seconds = (double(video_pts) * info.video_timebase.ToDouble()) + pts_offset_seconds;
1521
1522 // Debug output
1523 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number, "video_pts_seconds", video_pts_seconds);
1524}
1525
1526// Process an audio packet
1527void FFmpegReader::ProcessAudioPacket(int64_t requested_frame) {
1528 AudioLocation location;
1529 // Calculate location of current audio packet
1530 if (packet && packet->pts != AV_NOPTS_VALUE) {
1531 // Determine related video frame and starting sample # from audio PTS
1532 location = GetAudioPTSLocation(packet->pts);
1533
1534 // Track 1st audio packet after a successful seek
1535 if (!seek_audio_frame_found && is_seeking)
1536 seek_audio_frame_found = location.frame;
1537 }
1538
1539 // Create or get the existing frame object. Requested frame needs to be created
1540 // in working_cache at least once. Seek can clear the working_cache, so we must
1541 // add the requested frame back to the working_cache here. If it already exists,
1542 // it will be moved to the top of the working_cache.
1543 working_cache.Add(CreateFrame(requested_frame));
1544
1545 // Debug output
1546 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)",
1547 "requested_frame", requested_frame,
1548 "target_frame", location.frame,
1549 "starting_sample", location.sample_start);
1550
1551 // Init an AVFrame to hold the decoded audio samples
1552 int frame_finished = 0;
1553 AVFrame *audio_frame = AV_ALLOCATE_FRAME();
1554 AV_RESET_FRAME(audio_frame);
1555
1556 int packet_samples = 0;
1557 int data_size = 0;
1558
1559#if IS_FFMPEG_3_2
1560 int send_packet_err = avcodec_send_packet(aCodecCtx, packet);
1561 if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) {
1562 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Packet not sent)");
1563 }
1564 else {
1565 int receive_frame_err = avcodec_receive_frame(aCodecCtx, audio_frame);
1566 if (receive_frame_err >= 0) {
1567 frame_finished = 1;
1568 }
1569 if (receive_frame_err == AVERROR_EOF) {
1570 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (EOF detected from decoder)");
1571 packet_status.audio_eof = true;
1572 }
1573 if (receive_frame_err == AVERROR(EINVAL) || receive_frame_err == AVERROR_EOF) {
1574 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (invalid frame received or EOF from decoder)");
1575 avcodec_flush_buffers(aCodecCtx);
1576 }
1577 if (receive_frame_err != 0) {
1578 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (frame not ready yet from decoder)");
1579 }
1580 }
1581#else
1582 int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
1583#endif
1584
1585 if (frame_finished) {
1586 packet_status.audio_decoded++;
1587
1588 // This can be different than the current packet, so we need to look
1589 // at the current AVFrame from the audio decoder. This timestamp should
1590 // be used for the remainder of this function
1591 audio_pts = audio_frame->pts;
1592
1593 // Determine related video frame and starting sample # from audio PTS
1594 location = GetAudioPTSLocation(audio_pts);
1595
1596 // determine how many samples were decoded
1597 int plane_size = -1;
1598 data_size = av_samples_get_buffer_size(&plane_size,
1599 AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels,
1600 audio_frame->nb_samples,
1601 (AVSampleFormat) (AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1);
1602
1603 // Calculate total number of samples
1604 packet_samples = audio_frame->nb_samples * AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
1605 } else {
1606 if (audio_frame) {
1607 // Free audio frame
1608 AV_FREE_FRAME(&audio_frame);
1609 }
1610 }
1611
1612 // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
1613 int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
1614
1615 // Bail if no samples found
1616 if (pts_remaining_samples == 0) {
1617 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (No samples, bailing)",
1618 "packet_samples", packet_samples,
1619 "info.channels", info.channels,
1620 "pts_remaining_samples", pts_remaining_samples);
1621 return;
1622 }
1623
1624 while (pts_remaining_samples) {
1625 // Get Samples per frame (for this frame number)
1626 int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
1627
1628 // Calculate # of samples to add to this frame
1629 int samples = samples_per_frame - previous_packet_location.sample_start;
1630 if (samples > pts_remaining_samples)
1631 samples = pts_remaining_samples;
1632
1633 // Decrement remaining samples
1634 pts_remaining_samples -= samples;
1635
1636 if (pts_remaining_samples > 0) {
1637 // next frame
1638 previous_packet_location.frame++;
1639 previous_packet_location.sample_start = 0;
1640 } else {
1641 // Increment sample start
1642 previous_packet_location.sample_start += samples;
1643 }
1644 }
1645
1646 // Allocate audio buffer
1647 int16_t *audio_buf = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE + MY_INPUT_BUFFER_PADDING_SIZE];
1648
1649 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)",
1650 "packet_samples", packet_samples,
1651 "info.channels", info.channels,
1652 "info.sample_rate", info.sample_rate,
1653 "aCodecCtx->sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx),
1654 "AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16);
1655
1656 // Create output frame
1657 AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1658 AV_RESET_FRAME(audio_converted);
1659 audio_converted->nb_samples = audio_frame->nb_samples;
1660 av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1661
1662 SWRCONTEXT *avr = NULL;
1663 int nb_samples = 0;
1664
1665 // setup resample context
1666 avr = SWR_ALLOC();
1667 av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1668 av_opt_set_int(avr, "out_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1669 av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0);
1670 av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1671 av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
1672 av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1673 av_opt_set_int(avr, "in_channels", info.channels, 0);
1674 av_opt_set_int(avr, "out_channels", info.channels, 0);
1675 SWR_INIT(avr);
1676
1677 // Convert audio samples
1678 nb_samples = SWR_CONVERT(avr, // audio resample context
1679 audio_converted->data, // output data pointers
1680 audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1681 audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1682 audio_frame->data, // input data pointers
1683 audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1684 audio_frame->nb_samples); // number of input samples to convert
1685
1686 // Copy audio samples over original samples
1687 memcpy(audio_buf,
1688 audio_converted->data[0],
1689 static_cast<size_t>(audio_converted->nb_samples)
1690 * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)
1691 * info.channels);
1692
1693 // Deallocate resample buffer
1694 SWR_CLOSE(avr);
1695 SWR_FREE(&avr);
1696 avr = NULL;
1697
1698 // Free AVFrames
1699 av_free(audio_converted->data[0]);
1700 AV_FREE_FRAME(&audio_converted);
1701
1702 int64_t starting_frame_number = -1;
1703 bool partial_frame = true;
1704 for (int channel_filter = 0; channel_filter < info.channels; channel_filter++) {
1705 // Array of floats (to hold samples for each channel)
1706 starting_frame_number = location.frame;
1707 int channel_buffer_size = packet_samples / info.channels;
1708 float *channel_buffer = new float[channel_buffer_size];
1709
1710 // Init buffer array
1711 for (int z = 0; z < channel_buffer_size; z++)
1712 channel_buffer[z] = 0.0f;
1713
1714 // Loop through all samples and add them to our Frame based on channel.
1715 // Toggle through each channel number, since channel data is stored like (left right left right)
1716 int channel = 0;
1717 int position = 0;
1718 for (int sample = 0; sample < packet_samples; sample++) {
1719 // Only add samples for current channel
1720 if (channel_filter == channel) {
1721 // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
1722 channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1723
1724 // Increment audio position
1725 position++;
1726 }
1727
1728 // increment channel (if needed)
1729 if ((channel + 1) < info.channels)
1730 // move to next channel
1731 channel++;
1732 else
1733 // reset channel
1734 channel = 0;
1735 }
1736
1737 // Loop through samples, and add them to the correct frames
1738 int start = location.sample_start;
1739 int remaining_samples = channel_buffer_size;
1740 float *iterate_channel_buffer = channel_buffer; // pointer to channel buffer
1741 while (remaining_samples > 0) {
1742 // Get Samples per frame (for this frame number)
1743 int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number,
1745
1746 // Calculate # of samples to add to this frame
1747 int samples = samples_per_frame - start;
1748 if (samples > remaining_samples)
1749 samples = remaining_samples;
1750
1751 // Create or get the existing frame object
1752 std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1753
1754 // Determine if this frame was "partially" filled in
1755 if (samples_per_frame == start + samples)
1756 partial_frame = false;
1757 else
1758 partial_frame = true;
1759
1760 // Add samples for current channel to the frame.
1761 f->AddAudio(true, channel_filter, start, iterate_channel_buffer,
1762 samples, 1.0f);
1763
1764 // Debug output
1765 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)",
1766 "frame", starting_frame_number,
1767 "start", start,
1768 "samples", samples,
1769 "channel", channel_filter,
1770 "partial_frame", partial_frame,
1771 "samples_per_frame", samples_per_frame);
1772
1773 // Add or update cache
1774 working_cache.Add(f);
1775
1776 // Decrement remaining samples
1777 remaining_samples -= samples;
1778
1779 // Increment buffer (to next set of samples)
1780 if (remaining_samples > 0)
1781 iterate_channel_buffer += samples;
1782
1783 // Increment frame number
1784 starting_frame_number++;
1785
1786 // Reset starting sample #
1787 start = 0;
1788 }
1789
1790 // clear channel buffer
1791 delete[] channel_buffer;
1792 channel_buffer = NULL;
1793 iterate_channel_buffer = NULL;
1794 }
1795
1796 // Clean up some arrays
1797 delete[] audio_buf;
1798 audio_buf = NULL;
1799
1800 // Free audio frame
1801 AV_FREE_FRAME(&audio_frame);
1802
1803 // Get audio PTS in seconds
1804 audio_pts_seconds = (double(audio_pts) * info.audio_timebase.ToDouble()) + pts_offset_seconds;
1805
1806 // Debug output
1807 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)",
1808 "requested_frame", requested_frame,
1809 "starting_frame", location.frame,
1810 "end_frame", starting_frame_number - 1,
1811 "audio_pts_seconds", audio_pts_seconds);
1812
1813}
1814
1815
1816// Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
1817void FFmpegReader::Seek(int64_t requested_frame) {
1818 // Adjust for a requested frame that is too small or too large
1819 if (requested_frame < 1)
1820 requested_frame = 1;
1821 if (requested_frame > info.video_length)
1822 requested_frame = info.video_length;
1823 if (requested_frame > largest_frame_processed && packet_status.end_of_file) {
1824 // Not possible to search past largest_frame once EOF is reached (no more packets)
1825 return;
1826 }
1827
1828 // Debug output
1829 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek",
1830 "requested_frame", requested_frame,
1831 "seek_count", seek_count,
1832 "last_frame", last_frame);
1833
1834 // Clear working cache (since we are seeking to another location in the file)
1835 working_cache.Clear();
1836
1837 // Reset the last frame variable
1838 video_pts = 0.0;
1839 video_pts_seconds = NO_PTS_OFFSET;
1840 audio_pts = 0.0;
1841 audio_pts_seconds = NO_PTS_OFFSET;
1842 hold_packet = false;
1843 last_frame = 0;
1844 current_video_frame = 0;
1845 largest_frame_processed = 0;
1846 bool has_audio_override = info.has_audio;
1847 bool has_video_override = info.has_video;
1848
1849 // Init end-of-file detection variables
1850 packet_status.reset(false);
1851
1852 // Increment seek count
1853 seek_count++;
1854
1855 // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
1856 int buffer_amount = std::max(max_concurrent_frames, 8);
1857 if (requested_frame - buffer_amount < 20) {
1858 // prevent Open() from seeking again
1859 is_seeking = true;
1860
1861 // Close and re-open file (basically seeking to frame 1)
1862 Close();
1863 Open();
1864
1865 // Update overrides (since closing and re-opening might update these)
1866 info.has_audio = has_audio_override;
1867 info.has_video = has_video_override;
1868
1869 // Not actually seeking, so clear these flags
1870 is_seeking = false;
1871 if (seek_count == 1) {
1872 // Don't redefine this on multiple seek attempts for a specific frame
1873 seeking_frame = 1;
1874 seeking_pts = ConvertFrameToVideoPTS(1);
1875 }
1876 seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1877 seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1878
1879 } else {
1880 // Seek to nearest key-frame (aka, i-frame)
1881 bool seek_worked = false;
1882 int64_t seek_target = 0;
1883
1884 // Seek video stream (if any), except album arts
1885 if (!seek_worked && info.has_video && !HasAlbumArt()) {
1886 seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1887 if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1888 fprintf(stderr, "%s: error while seeking video stream\n", pFormatCtx->AV_FILENAME);
1889 } else {
1890 // VIDEO SEEK
1891 is_video_seek = true;
1892 seek_worked = true;
1893 }
1894 }
1895
1896 // Seek audio stream (if not already seeked... and if an audio stream is found)
1897 if (!seek_worked && info.has_audio) {
1898 seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1899 if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1900 fprintf(stderr, "%s: error while seeking audio stream\n", pFormatCtx->AV_FILENAME);
1901 } else {
1902 // AUDIO SEEK
1903 is_video_seek = false;
1904 seek_worked = true;
1905 }
1906 }
1907
1908 // Was the seek successful?
1909 if (seek_worked) {
1910 // Flush audio buffer
1911 if (info.has_audio)
1912 avcodec_flush_buffers(aCodecCtx);
1913
1914 // Flush video buffer
1915 if (info.has_video)
1916 avcodec_flush_buffers(pCodecCtx);
1917
1918 // Reset previous audio location to zero
1919 previous_packet_location.frame = -1;
1920 previous_packet_location.sample_start = 0;
1921
1922 // init seek flags
1923 is_seeking = true;
1924 if (seek_count == 1) {
1925 // Don't redefine this on multiple seek attempts for a specific frame
1926 seeking_pts = seek_target;
1927 seeking_frame = requested_frame;
1928 }
1929 seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1930 seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1931
1932 } else {
1933 // seek failed
1934 seeking_pts = 0;
1935 seeking_frame = 0;
1936
1937 // prevent Open() from seeking again
1938 is_seeking = true;
1939
1940 // Close and re-open file (basically seeking to frame 1)
1941 Close();
1942 Open();
1943
1944 // Not actually seeking, so clear these flags
1945 is_seeking = false;
1946
1947 // disable seeking for this reader (since it failed)
1948 enable_seek = false;
1949
1950 // Update overrides (since closing and re-opening might update these)
1951 info.has_audio = has_audio_override;
1952 info.has_video = has_video_override;
1953 }
1954 }
1955}
1956
1957// Get the PTS for the current video packet
1958int64_t FFmpegReader::GetPacketPTS() {
1959 if (packet) {
1960 int64_t current_pts = packet->pts;
1961 if (current_pts == AV_NOPTS_VALUE && packet->dts != AV_NOPTS_VALUE)
1962 current_pts = packet->dts;
1963
1964 // Return adjusted PTS
1965 return current_pts;
1966 } else {
1967 // No packet, return NO PTS
1968 return AV_NOPTS_VALUE;
1969 }
1970}
1971
1972// Update PTS Offset (if any)
1973void FFmpegReader::UpdatePTSOffset() {
1974 if (pts_offset_seconds != NO_PTS_OFFSET) {
1975 // Skip this method if we have already set PTS offset
1976 return;
1977 }
1978 pts_offset_seconds = 0.0;
1979 double video_pts_offset_seconds = 0.0;
1980 double audio_pts_offset_seconds = 0.0;
1981
1982 bool has_video_pts = false;
1983 if (!info.has_video) {
1984 // Mark as checked
1985 has_video_pts = true;
1986 }
1987 bool has_audio_pts = false;
1988 if (!info.has_audio) {
1989 // Mark as checked
1990 has_audio_pts = true;
1991 }
1992
1993 // Loop through the stream (until a packet from all streams is found)
1994 while (!has_video_pts || !has_audio_pts) {
1995 // Get the next packet (if any)
1996 if (GetNextPacket() < 0)
1997 // Break loop when no more packets found
1998 break;
1999
2000 // Get PTS of this packet
2001 int64_t pts = GetPacketPTS();
2002
2003 // Video packet
2004 if (!has_video_pts && packet->stream_index == videoStream) {
2005 // Get the video packet start time (in seconds)
2006 video_pts_offset_seconds = 0.0 - (video_pts * info.video_timebase.ToDouble());
2007
2008 // Is timestamp close to zero (within X seconds)
2009 // Ignore wildly invalid timestamps (i.e. -234923423423)
2010 if (std::abs(video_pts_offset_seconds) <= 10.0) {
2011 has_video_pts = true;
2012 }
2013 }
2014 else if (!has_audio_pts && packet->stream_index == audioStream) {
2015 // Get the audio packet start time (in seconds)
2016 audio_pts_offset_seconds = 0.0 - (pts * info.audio_timebase.ToDouble());
2017
2018 // Is timestamp close to zero (within X seconds)
2019 // Ignore wildly invalid timestamps (i.e. -234923423423)
2020 if (std::abs(audio_pts_offset_seconds) <= 10.0) {
2021 has_audio_pts = true;
2022 }
2023 }
2024 }
2025
2026 // Do we have all valid timestamps to determine PTS offset?
2027 if (has_video_pts && has_audio_pts) {
2028 // Set PTS Offset to the smallest offset
2029 // [ video timestamp ]
2030 // [ audio timestamp ]
2031 //
2032 // ** SHIFT TIMESTAMPS TO ZERO **
2033 //
2034 //[ video timestamp ]
2035 // [ audio timestamp ]
2036 //
2037 // Since all offsets are negative at this point, we want the max value, which
2038 // represents the closest to zero
2039 pts_offset_seconds = std::max(video_pts_offset_seconds, audio_pts_offset_seconds);
2040 }
2041}
2042
2043// Convert PTS into Frame Number
2044int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) {
2045 // Apply PTS offset
2046 int64_t previous_video_frame = current_video_frame;
2047
2048 // Get the video packet start time (in seconds)
2049 double video_seconds = (double(pts) * info.video_timebase.ToDouble()) + pts_offset_seconds;
2050
2051 // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
2052 int64_t frame = round(video_seconds * info.fps.ToDouble()) + 1;
2053
2054 // Keep track of the expected video frame #
2055 if (current_video_frame == 0)
2056 current_video_frame = frame;
2057 else {
2058
2059 // Sometimes frames are duplicated due to identical (or similar) timestamps
2060 if (frame == previous_video_frame) {
2061 // return -1 frame number
2062 frame = -1;
2063 } else {
2064 // Increment expected frame
2065 current_video_frame++;
2066 }
2067 }
2068
2069 // Return frame #
2070 return frame;
2071}
2072
2073// Convert Frame Number into Video PTS
2074int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number) {
2075 // Get timestamp of this frame (in seconds)
2076 double seconds = (double(frame_number - 1) / info.fps.ToDouble()) + pts_offset_seconds;
2077
2078 // Calculate the # of video packets in this timestamp
2079 int64_t video_pts = round(seconds / info.video_timebase.ToDouble());
2080
2081 // Apply PTS offset (opposite)
2082 return video_pts;
2083}
2084
2085// Convert Frame Number into Video PTS
2086int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number) {
2087 // Get timestamp of this frame (in seconds)
2088 double seconds = (double(frame_number - 1) / info.fps.ToDouble()) + pts_offset_seconds;
2089
2090 // Calculate the # of audio packets in this timestamp
2091 int64_t audio_pts = round(seconds / info.audio_timebase.ToDouble());
2092
2093 // Apply PTS offset (opposite)
2094 return audio_pts;
2095}
2096
2097// Calculate Starting video frame and sample # for an audio PTS
2098AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) {
2099 // Get the audio packet start time (in seconds)
2100 double audio_seconds = (double(pts) * info.audio_timebase.ToDouble()) + pts_offset_seconds;
2101
2102 // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
2103 double frame = (audio_seconds * info.fps.ToDouble()) + 1;
2104
2105 // Frame # as a whole number (no more decimals)
2106 int64_t whole_frame = int64_t(frame);
2107
2108 // Remove the whole number, and only get the decimal of the frame
2109 double sample_start_percentage = frame - double(whole_frame);
2110
2111 // Get Samples per frame
2112 int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
2113
2114 // Calculate the sample # to start on
2115 int sample_start = round(double(samples_per_frame) * sample_start_percentage);
2116
2117 // Protect against broken (i.e. negative) timestamps
2118 if (whole_frame < 1)
2119 whole_frame = 1;
2120 if (sample_start < 0)
2121 sample_start = 0;
2122
2123 // Prepare final audio packet location
2124 AudioLocation location = {whole_frame, sample_start};
2125
2126 // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
2127 if (previous_packet_location.frame != -1) {
2128 if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame)) {
2129 int64_t orig_frame = location.frame;
2130 int orig_start = location.sample_start;
2131
2132 // Update sample start, to prevent gaps in audio
2133 location.sample_start = previous_packet_location.sample_start;
2134 location.frame = previous_packet_location.frame;
2135
2136 // Debug output
2137 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2138
2139 } else {
2140 // Debug output
2141 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2142 }
2143 }
2144
2145 // Set previous location
2146 previous_packet_location = location;
2147
2148 // Return the associated video frame and starting sample #
2149 return location;
2150}
2151
2152// Create a new Frame (or return an existing one) and add it to the working queue.
2153std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame) {
2154 // Check working cache
2155 std::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
2156
2157 if (!output) {
2158 // (re-)Check working cache
2159 output = working_cache.GetFrame(requested_frame);
2160 if(output) return output;
2161
2162 // Create a new frame on the working cache
2163 output = std::make_shared<Frame>(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels);
2164 output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
2165 output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
2166 output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
2167
2168 working_cache.Add(output);
2169
2170 // Set the largest processed frame (if this is larger)
2171 if (requested_frame > largest_frame_processed)
2172 largest_frame_processed = requested_frame;
2173 }
2174 // Return frame
2175 return output;
2176}
2177
2178// Determine if frame is partial due to seek
2179bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
2180
2181 // Sometimes a seek gets partial frames, and we need to remove them
2182 bool seek_trash = false;
2183 int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
2184 if (seek_video_frame_found > max_seeked_frame) {
2185 max_seeked_frame = seek_video_frame_found;
2186 }
2187 if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
2188 (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame)) {
2189 seek_trash = true;
2190 }
2191
2192 return seek_trash;
2193}
2194
2195// Check the working queue, and move finished frames to the finished queue
2196void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) {
2197
2198 // Prevent async calls to the following code
2199 const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
2200
2201 // Get a list of current working queue frames in the cache (in-progress frames)
2202 std::vector<std::shared_ptr<openshot::Frame>> working_frames = working_cache.GetFrames();
2203 std::vector<std::shared_ptr<openshot::Frame>>::iterator working_itr;
2204
2205 // Loop through all working queue frames (sorted by frame #)
2206 for(working_itr = working_frames.begin(); working_itr != working_frames.end(); ++working_itr)
2207 {
2208 // Get working frame
2209 std::shared_ptr<Frame> f = *working_itr;
2210
2211 // Was a frame found? Is frame requested yet?
2212 if (!f || f->number > requested_frame) {
2213 // If not, skip to next one
2214 continue;
2215 }
2216
2217 // Calculate PTS in seconds (of working frame), and the most recent processed pts value
2218 double frame_pts_seconds = (double(f->number - 1) / info.fps.ToDouble()) + pts_offset_seconds;
2219 double recent_pts_seconds = std::max(video_pts_seconds, audio_pts_seconds);
2220
2221 // Determine if video and audio are ready (based on timestamps)
2222 bool is_video_ready = false;
2223 bool is_audio_ready = false;
2224 double recent_pts_diff = recent_pts_seconds - frame_pts_seconds;
2225 if ((frame_pts_seconds <= video_pts_seconds)
2226 || (recent_pts_diff > 1.5)
2227 || packet_status.video_eof || packet_status.end_of_file) {
2228 // Video stream is past this frame (so it must be done)
2229 // OR video stream is too far behind, missing, or end-of-file
2230 is_video_ready = true;
2231 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (video ready)",
2232 "frame_number", f->number,
2233 "frame_pts_seconds", frame_pts_seconds,
2234 "video_pts_seconds", video_pts_seconds,
2235 "recent_pts_diff", recent_pts_diff);
2236 if (info.has_video && !f->has_image_data) {
2237 // Frame has no image data (copy from previous frame)
2238 // Loop backwards through final frames (looking for the nearest, previous frame image)
2239 for (int64_t previous_frame = requested_frame - 1; previous_frame > 0; previous_frame--) {
2240 std::shared_ptr<Frame> previous_frame_instance = final_cache.GetFrame(previous_frame);
2241 if (previous_frame_instance && previous_frame_instance->has_image_data) {
2242 // Copy image from last decoded frame
2243 f->AddImage(std::make_shared<QImage>(previous_frame_instance->GetImage()->copy()));
2244 break;
2245 }
2246 }
2247
2248 if (last_video_frame && !f->has_image_data) {
2249 // Copy image from last decoded frame
2250 f->AddImage(std::make_shared<QImage>(last_video_frame->GetImage()->copy()));
2251 } else if (!f->has_image_data) {
2252 f->AddColor("#000000");
2253 }
2254 }
2255 }
2256
2257 double audio_pts_diff = audio_pts_seconds - frame_pts_seconds;
2258 if ((frame_pts_seconds < audio_pts_seconds && audio_pts_diff > 1.0)
2259 || (recent_pts_diff > 1.5)
2260 || packet_status.audio_eof || packet_status.end_of_file) {
2261 // Audio stream is past this frame (so it must be done)
2262 // OR audio stream is too far behind, missing, or end-of-file
2263 // Adding a bit of margin here, to allow for partial audio packets
2264 is_audio_ready = true;
2265 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (audio ready)",
2266 "frame_number", f->number,
2267 "frame_pts_seconds", frame_pts_seconds,
2268 "audio_pts_seconds", audio_pts_seconds,
2269 "audio_pts_diff", audio_pts_diff,
2270 "recent_pts_diff", recent_pts_diff);
2271 }
2272 bool is_seek_trash = IsPartialFrame(f->number);
2273
2274 // Adjust for available streams
2275 if (!info.has_video) is_video_ready = true;
2276 if (!info.has_audio) is_audio_ready = true;
2277
2278 // Debug output
2279 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames",
2280 "frame_number", f->number,
2281 "is_video_ready", is_video_ready,
2282 "is_audio_ready", is_audio_ready,
2283 "video_eof", packet_status.video_eof,
2284 "audio_eof", packet_status.audio_eof,
2285 "end_of_file", packet_status.end_of_file);
2286
2287 // Check if working frame is final
2288 if ((!packet_status.end_of_file && is_video_ready && is_audio_ready) || packet_status.end_of_file || is_seek_trash) {
2289 // Debug output
2290 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)",
2291 "requested_frame", requested_frame,
2292 "f->number", f->number,
2293 "is_seek_trash", is_seek_trash,
2294 "Working Cache Count", working_cache.Count(),
2295 "Final Cache Count", final_cache.Count(),
2296 "end_of_file", packet_status.end_of_file);
2297
2298 if (!is_seek_trash) {
2299 // Move frame to final cache
2300 final_cache.Add(f);
2301
2302 // Remove frame from working cache
2303 working_cache.Remove(f->number);
2304
2305 // Update last frame processed
2306 last_frame = f->number;
2307 } else {
2308 // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
2309 working_cache.Remove(f->number);
2310 }
2311
2312 }
2313 }
2314
2315 // Clear vector of frames
2316 working_frames.clear();
2317 working_frames.shrink_to_fit();
2318}
2319
2320// Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
2321void FFmpegReader::CheckFPS() {
2322 if (check_fps) {
2323 // Do not check FPS more than 1 time
2324 return;
2325 } else {
2326 check_fps = true;
2327 }
2328
2329 int frames_per_second[3] = {0,0,0};
2330 int max_fps_index = sizeof(frames_per_second) / sizeof(frames_per_second[0]);
2331 int fps_index = 0;
2332
2333 int all_frames_detected = 0;
2334 int starting_frames_detected = 0;
2335
2336 // Loop through the stream
2337 while (true) {
2338 // Get the next packet (if any)
2339 if (GetNextPacket() < 0)
2340 // Break loop when no more packets found
2341 break;
2342
2343 // Video packet
2344 if (packet->stream_index == videoStream) {
2345 // Get the video packet start time (in seconds)
2346 double video_seconds = (double(GetPacketPTS()) * info.video_timebase.ToDouble()) + pts_offset_seconds;
2347 fps_index = int(video_seconds); // truncate float timestamp to int (second 1, second 2, second 3)
2348
2349 // Is this video packet from the first few seconds?
2350 if (fps_index >= 0 && fps_index < max_fps_index) {
2351 // Yes, keep track of how many frames per second (over the first few seconds)
2352 starting_frames_detected++;
2353 frames_per_second[fps_index]++;
2354 }
2355
2356 // Track all video packets detected
2357 all_frames_detected++;
2358 }
2359 }
2360
2361 // Calculate FPS (based on the first few seconds of video packets)
2362 float avg_fps = 30.0;
2363 if (starting_frames_detected > 0 && fps_index > 0) {
2364 avg_fps = float(starting_frames_detected) / std::min(fps_index, max_fps_index);
2365 }
2366
2367 // Verify average FPS is a reasonable value
2368 if (avg_fps < 8.0) {
2369 // Invalid FPS assumed, so switching to a sane default FPS instead
2370 avg_fps = 30.0;
2371 }
2372
2373 // Update FPS (truncate average FPS to Integer)
2374 info.fps = Fraction(int(avg_fps), 1);
2375
2376 // Update Duration and Length
2377 if (all_frames_detected > 0) {
2378 // Use all video frames detected to calculate # of frames
2379 info.video_length = all_frames_detected;
2380 info.duration = all_frames_detected / avg_fps;
2381 } else {
2382 // Use previous duration to calculate # of frames
2383 info.video_length = info.duration * avg_fps;
2384 }
2385
2386 // Update video bit rate
2388}
2389
2390// Remove AVFrame from cache (and deallocate its memory)
2391void FFmpegReader::RemoveAVFrame(AVFrame *remove_frame) {
2392 // Remove pFrame (if exists)
2393 if (remove_frame) {
2394 // Free memory
2395 av_freep(&remove_frame->data[0]);
2396#ifndef WIN32
2397 AV_FREE_FRAME(&remove_frame);
2398#endif
2399 }
2400}
2401
2402// Remove AVPacket from cache (and deallocate its memory)
2403void FFmpegReader::RemoveAVPacket(AVPacket *remove_packet) {
2404 // deallocate memory for packet
2405 AV_FREE_PACKET(remove_packet);
2406
2407 // Delete the object
2408 delete remove_packet;
2409}
2410
2411// Generate JSON string of this object
2412std::string FFmpegReader::Json() const {
2413
2414 // Return formatted string
2415 return JsonValue().toStyledString();
2416}
2417
2418// Generate Json::Value for this object
2419Json::Value FFmpegReader::JsonValue() const {
2420
2421 // Create root json object
2422 Json::Value root = ReaderBase::JsonValue(); // get parent properties
2423 root["type"] = "FFmpegReader";
2424 root["path"] = path;
2425
2426 // return JsonValue
2427 return root;
2428}
2429
2430// Load JSON string into this object
2431void FFmpegReader::SetJson(const std::string value) {
2432
2433 // Parse JSON string into JSON objects
2434 try {
2435 const Json::Value root = openshot::stringToJson(value);
2436 // Set all values that match
2437 SetJsonValue(root);
2438 }
2439 catch (const std::exception& e) {
2440 // Error parsing JSON (or missing keys)
2441 throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
2442 }
2443}
2444
2445// Load Json::Value into this object
2446void FFmpegReader::SetJsonValue(const Json::Value root) {
2447
2448 // Set parent data
2450
2451 // Set data from Json (if key is found)
2452 if (!root["path"].isNull())
2453 path = root["path"].asString();
2454
2455 // Re-Open path, and re-init everything (if needed)
2456 if (is_open) {
2457 Close();
2458 Open();
2459 }
2460}
Header file for all Exception classes.
AVPixelFormat hw_de_av_pix_fmt_global
AVHWDeviceType hw_de_av_device_type_global
int hw_de_on
Header file for FFmpegReader class.
Header file for FFmpegUtilities.
#define AV_FREE_CONTEXT(av_context)
#define SWR_INIT(ctx)
#define AV_FREE_FRAME(av_frame)
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
#define SWR_ALLOC()
#define SWR_CLOSE(ctx)
#define AV_GET_CODEC_TYPE(av_stream)
#define PixelFormat
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
#define AV_GET_CODEC_CONTEXT(av_stream, av_codec)
#define AV_FIND_DECODER_CODEC_ID(av_stream)
#define AV_ALLOCATE_FRAME()
#define AV_REGISTER_ALL
#define PIX_FMT_RGBA
#define SWR_FREE(ctx)
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
#define AV_FREE_PACKET(av_packet)
#define SWRCONTEXT
#define AVCODEC_REGISTER_ALL
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
#define MY_INPUT_BUFFER_PADDING_SIZE
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
#define AV_RESET_FRAME(av_frame)
#define FF_NUM_PROCESSORS
#define OPEN_MP_NUM_PROCESSORS
Header file for Timeline class.
Header file for ZeroMQ-based Logger class.
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:30
int64_t Count()
Count the frames in the queue.
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
Definition: CacheMemory.cpp:46
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheMemory.cpp:80
std::vector< std::shared_ptr< openshot::Frame > > GetFrames()
Get an array of all Frames.
Definition: CacheMemory.cpp:96
void Remove(int64_t frame_number)
Remove a specific frame.
void Clear()
Clear the cache of all frames.
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:89
openshot::Keyframe scale_x
Curve representing the horizontal scaling in percent (0 to 1)
Definition: Clip.h:298
openshot::TimelineBase * ParentTimeline() override
Get the associated Timeline pointer (if any)
Definition: Clip.h:276
openshot::Keyframe scale_y
Curve representing the vertical scaling in percent (0 to 1)
Definition: Clip.h:299
openshot::ScaleType scale
The scale determines how a clip should be resized to fit its parent.
Definition: Clip.h:166
double Y
The Y value of the coordinate (usually representing the value of the property being animated)
Definition: Coordinate.h:41
void Open() override
Open File - which is called by the constructor automatically.
FFmpegReader(const std::string &path, bool inspect_reader=true)
Constructor for FFmpegReader.
Json::Value JsonValue() const override
Generate Json::Value for this object.
bool GetIsDurationKnown()
Return true if frame can be read with GetFrame()
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:228
virtual ~FFmpegReader()
Destructor.
std::string Json() const override
Generate JSON string of this object.
std::shared_ptr< openshot::Frame > GetFrame(int64_t requested_frame) override
void Close() override
Close File.
void SetJson(const std::string value) override
Load JSON string into this object.
This class represents a fraction.
Definition: Fraction.h:30
int num
Numerator for the fraction.
Definition: Fraction.h:32
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:35
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:40
int den
Denominator for the fraction.
Definition: Fraction.h:33
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:480
Exception when no valid codec is found for a file.
Definition: Exceptions.h:173
Exception for files that can not be found or opened.
Definition: Exceptions.h:188
Exception for invalid JSON.
Definition: Exceptions.h:218
Point GetMaxPoint() const
Get max point (by Y coordinate)
Definition: KeyFrame.cpp:245
Exception when no streams are found in the file.
Definition: Exceptions.h:286
Exception when memory could not be allocated.
Definition: Exceptions.h:349
Coordinate co
This is the primary coordinate.
Definition: Point.h:66
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:88
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: ReaderBase.cpp:162
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
Definition: ReaderBase.cpp:107
std::recursive_mutex getFrameMutex
Mutex for multiple threads.
Definition: ReaderBase.h:79
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
Definition: ReaderBase.cpp:245
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:364
int DE_LIMIT_WIDTH_MAX
Maximum columns that hardware decode can handle.
Definition: Settings.h:77
int HW_DE_DEVICE_SET
Which GPU to use to decode (0 is the first)
Definition: Settings.h:80
int DE_LIMIT_HEIGHT_MAX
Maximum rows that hardware decode can handle.
Definition: Settings.h:74
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: Settings.cpp:23
int HARDWARE_DECODER
Use video codec for faster video decoding (if supported)
Definition: Settings.h:62
int preview_height
Optional preview width of timeline image. If your preview window is smaller than the timeline,...
Definition: TimelineBase.h:44
int preview_width
Optional preview width of timeline image. If your preview window is smaller than the timeline,...
Definition: TimelineBase.h:43
This class represents a timeline.
Definition: Timeline.h:150
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:178
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: ZmqLogger.cpp:35
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:29
@ SCALE_FIT
Scale the clip until either height or width fills the canvas (with no cropping)
Definition: Enums.h:38
@ SCALE_STRETCH
Scale the clip until both height and width fill the canvas (distort to fit)
Definition: Enums.h:39
@ SCALE_CROP
Scale the clip until both height and width fill the canvas (cropping the overlap)
Definition: Enums.h:37
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: AudioLocation.h:25
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
void reset(bool eof)
Definition: FFmpegReader.h:68
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:59
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:49
bool has_single_image
Determines if this file only contains a single image.
Definition: ReaderBase.h:42
float duration
Length of time (in seconds)
Definition: ReaderBase.h:43
openshot::Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:64
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:46
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:61
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:48
openshot::Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3)
Definition: ReaderBase.h:51
int height
The height of the video (in pixels)
Definition: ReaderBase.h:45
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:47
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:53
std::string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:58
std::map< std::string, std::string > metadata
An optional map/dictionary of metadata for this reader.
Definition: ReaderBase.h:65
std::string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:52
openshot::Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square)
Definition: ReaderBase.h:50
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:62
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:40
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:41
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:55
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:54
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:60
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:63
int64_t file_size
Size of file (in bytes)
Definition: ReaderBase.h:44