Piratux

Untitled

Feb 12th, 2022
739
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
C++ 41.70 KB | None | 0 0
  1. // TODO:
  2. // - Make sure video frames don't get overwritten (avoid re-allocating, because
  3. // that will be hard to manage, instead pop old frames if they are behind the audio)
  4. // - Change returns to exits in some places (change the assert macro)
  5. // - Add stop audio/video
  6. // - Add volume control
  7. // - Add video seeking
  8. // - Check if video/audio is opened before every function related to video/audio
  9. // - Create debug print information functions for video/audio
  10.  
  11. #ifndef OLCPGEX_MEDIA_H
  12. #define OLCPGEX_MEDIA_H
  13.  
  14. extern "C" {
  15. // Video and audio dependencies
  16. #include <libavcodec/avcodec.h>
  17. #include <libavformat/avformat.h>
  18.  
  19. // Video dependencies
  20. #include <libswscale/swscale.h>
  21. #include <inttypes.h>
  22. #include <libavutil/pixdesc.h>
  23.  
  24. // Audio dependencies
  25. #include <libavutil/avutil.h>
  26. #include <libswresample/swresample.h>
  27. #include <libavutil/audio_fifo.h>
  28. }
  29.  
  30. #define MINIAUDIO_IMPLEMENTATION
  31. #include "miniaudio.h"
  32.  
  33.  
  34. #include <iostream>
  35. #include <string>
  36. #include <cstdint>
  37. #include <stdexcept>
  38. #include <thread>
  39. #include <mutex>
  40. #include <condition_variable>
  41. #include <queue>
  42.  
  43.  
  44. // This definition usually gets set automatically by the IDEs
  45. #ifdef NDEBUG
  46. // Release macro
  47. #define OLC_MEDIA_ASSERT(condition, return_value, message) if(!(condition)){return return_value;}
  48. #else
  49. // Debug macro
  50. #define OLC_MEDIA_ASSERT(condition, return_value, message) if(!(condition)){std::cerr << "Assert failed: \"" << __FILE__ << "\" (" << __LINE__ << ")\n" << message << '\n'; return return_value;}
  51. #endif
  52.  
  53. // Declarations
  54. namespace olc {
  55.  
  56.     enum class MediaResult {
  57.         Success = 0,
  58.         Error = 1,
  59.     };
  60.  
  61.     // Thread safe "queue" that uses circular buffer
  62.     class VideoQueue {
  63.     private:
  64.         uint16_t _size = 0;
  65.         uint16_t _capacity = 0;
  66.        
  67.         uint16_t _insert_idx = 0; // idx that points to location where new element will be inserted
  68.         uint16_t _delete_idx = 0; // idx that points to location where oldest element will be deleted from
  69.  
  70.         AVFrame** _data = nullptr;
  71.  
  72.         mutable std::mutex _mut;
  73.  
  74.     public:
  75.         VideoQueue() {
  76.         }
  77.  
  78.         ~VideoQueue() {
  79.             clear();
  80.             free();
  81.         }
  82.  
  83.         // Suggested to set capacity to fps (but min capacity must be 2)
  84.         MediaResult init(uint16_t capacity) {
  85.             // Always reset the values
  86.             _size = 0;
  87.             _insert_idx = 0;
  88.             _delete_idx = 0;
  89.  
  90.             // If video fifo was already in use, reset it first
  91.             if (_data != nullptr) {
  92.                 clear();
  93.                 free();
  94.             }
  95.  
  96.             if(capacity <= 1)
  97.                 return MediaResult::Error;
  98.  
  99.             _data = new AVFrame * [capacity];
  100.             if (_data == nullptr)
  101.                 return MediaResult::Error;
  102.  
  103.             _capacity = capacity;
  104.             for (uint16_t i = 0; i < _capacity; ++i) {
  105.                 _data[i] = av_frame_alloc();
  106.  
  107.                 // Memory might run out if capacity is too big (which might happen if video fps is insanely large)
  108.                 if (_data[i] == nullptr)
  109.                     return MediaResult::Error;
  110.             }
  111.  
  112.             return MediaResult::Success;
  113.         }
  114.  
  115.         AVFrame* back() const {
  116.             std::unique_lock<std::mutex> lock(_mut);
  117.             return _data[_insert_idx];
  118.         }
  119.  
  120.         AVFrame* front() const {
  121.             std::unique_lock<std::mutex> lock(_mut);
  122.             return _data[_delete_idx];
  123.         }
  124.  
  125.         // Push updated AVFrame from "back()"
  126.         void push() {
  127.             std::unique_lock<std::mutex> lock(_mut);
  128.  
  129.             _insert_idx = (_insert_idx + 1) % _capacity;
  130.             ++_size;
  131.         }
  132.  
  133.         // Pop AVFrame from the front() unreferencing it
  134.         // If size is 0, does nothing
  135.         void pop() {
  136.             std::unique_lock<std::mutex> lock(_mut);
  137.  
  138.             if (_size > 0) {
  139.                 av_frame_unref(_data[_delete_idx]);
  140.  
  141.                 _delete_idx = (_delete_idx + 1) % _capacity;
  142.                 --_size;
  143.             }
  144.         }
  145.  
  146.         size_t size() const {
  147.             std::unique_lock<std::mutex> lock(_mut);
  148.             return _size;
  149.         }
  150.  
  151.         size_t capacity() const {
  152.             std::unique_lock<std::mutex> lock(_mut);
  153.             return _capacity;
  154.         }
  155.  
  156.         void clear() {
  157.             if (_data != nullptr) {
  158.                 for (uint16_t i = 0; i < _capacity; ++i) {
  159.                     av_frame_unref(_data[i]);
  160.                 }
  161.             }
  162.         }
  163.  
  164.         // De-allocates fifo structure
  165.         void free() {
  166.             if (_data != nullptr) {
  167.                 for (uint16_t i = 0; i < _capacity; ++i) {
  168.                     av_frame_free(&_data[i]);
  169.                 }
  170.  
  171.                 delete[] _data;
  172.                 _data = nullptr;
  173.             }            
  174.         }
  175.     };
  176.  
  177.     // Thread safe wrapper for audio fifo read/write
  178.     class AudioQueue {
  179.     private:
  180.         AVAudioFifo* _fifo = nullptr;
  181.         mutable std::mutex _mut;
  182.  
  183.     public:
  184.         AudioQueue() {
  185.         }
  186.  
  187.         ~AudioQueue() {
  188.             clear();
  189.             free();
  190.         }
  191.  
  192.         // Suggested to set capacity to sample rate
  193.         MediaResult init(int channels, int capacity) {
  194.             // If these parameters are 0, there is something wrong
  195.             if (capacity == 0 || channels == 0)
  196.                 return MediaResult::Error;
  197.  
  198.             if (_fifo != nullptr) {
  199.                 clear();
  200.                 free();
  201.             }
  202.  
  203.             _fifo = av_audio_fifo_alloc(AV_SAMPLE_FMT_FLT, channels, capacity);
  204.  
  205.             if (_fifo == nullptr)
  206.                 return MediaResult::Error;
  207.  
  208.             return MediaResult::Success;
  209.         }
  210.  
  211.         int push(void** data, int samples) {
  212.             std::lock_guard<std::mutex> lock(_mut);
  213.  
  214.             int space = av_audio_fifo_space(_fifo);
  215.  
  216.             // If capacity is reached, drain some audio samples
  217.             if (samples > space) {
  218.                 av_audio_fifo_drain(_fifo, std::min(av_audio_fifo_size(_fifo), samples - space));
  219.             }
  220.                
  221.  
  222.             return av_audio_fifo_write(_fifo, data, samples);
  223.         }
  224.  
  225.         int pop(void** data, int samples) {
  226.             std::unique_lock<std::mutex> lock(_mut);
  227.             return av_audio_fifo_read(_fifo, data, samples);
  228.         }
  229.  
  230.         void drain(int samples) {
  231.             std::unique_lock<std::mutex> lock(_mut);
  232.             av_audio_fifo_drain(_fifo, samples);
  233.         }
  234.  
  235.         int size() {
  236.             std::unique_lock<std::mutex> lock(_mut);
  237.             return av_audio_fifo_size(_fifo);
  238.         }
  239.  
  240.         int capacity() {
  241.             std::unique_lock<std::mutex> lock(_mut);
  242.             return av_audio_fifo_space(_fifo) + av_audio_fifo_size(_fifo);
  243.         }
  244.  
  245.         // Empties out all the frames
  246.         void clear() {
  247.             if(_fifo != nullptr)
  248.                 av_audio_fifo_drain(_fifo, av_audio_fifo_size(_fifo));
  249.         }
  250.  
  251.         // De-allocates fifo structure
  252.         void free() {
  253.             av_audio_fifo_free(_fifo);
  254.             _fifo = nullptr;
  255.         }
  256.     };
  257.  
  258.     class Media {
  259.     private:
  260.         // -- Private internal state --
  261.         AVFormatContext* av_format_ctx = nullptr;
  262.  
  263.         uint16_t preloaded_frames_scale;
  264.  
  265.         // If false, video capture wasn't opened, or last frame was put into queue
  266.         std::atomic<bool> finished_reading = false;
  267.  
  268.         // When true, the frame loading thread keeps on working
  269.         // When set to false, and conditional is called, frame loading thread is halted
  270.         std::atomic<bool> keep_loading = true;
  271.        
  272.         std::thread frame_loader;
  273.         std::mutex mutex;
  274.         std::condition_variable conditional;
  275.  
  276.  
  277.         // -- Video stuff --
  278.         int video_stream_index = -1;
  279.         AVRational video_time_base;
  280.         VideoQueue video_fifo;
  281.         const AVCodec* av_video_codec = nullptr;
  282.         AVCodecContext* av_video_codec_ctx = nullptr;
  283.         SwsContext* sws_video_scaler_ctx = nullptr;
  284.         olc::Renderable video_frame;
  285.         int video_width = 0;
  286.         int video_height = 0;
  287.         int video_delay = 0;
  288.         float delta_time_accumulator = 0.0f;
  289.         double last_video_pts = 0.0;
  290.         bool video_opened = false;
  291.  
  292.  
  293.         // -- Audio stuff --
  294.         int audio_stream_index = -1;
  295.         AVRational audio_time_base;
  296.         AudioQueue audio_fifo;
  297.         const AVCodec* av_audio_codec = nullptr;
  298.         AVCodecContext* av_audio_codec_ctx = nullptr;
  299.         SwrContext* swr_audio_resampler = nullptr;
  300.         ma_device audio_device;
  301.         size_t audio_frames_consumed = 0;
  302.         std::atomic<double> audio_time = 0.0;
  303.         int audio_sample_size = 0;
  304.         int audio_channel_count = 0;
  305.         bool audio_opened = false;
  306.  
  307.     public:
  308.         Media();
  309.         ~Media();
  310.  
  311.         // -- Video and audio functions --
  312.  
  313.         // If media is already open, closes it first.
  314.         // preloaded_frames_scale: Affects how many seconds of pre-decoded video/audio frames should be stored. (only suggested to
  315.         // increase it, when you notice that some of your video/audio frames are skipped)
  316.         MediaResult Open(const char* filename, bool open_video, bool open_audio, uint16_t preloaded_frames_scale = 1);
  317.         void Close();
  318.         // Returns true if any of the following is true:
  319.         // - Media file wasn't opened;
  320.         // - No more frames are available in the media file;
  321.         // - Error occured when trying to receive a packet.
  322.         bool FinishedReading();
  323.  
  324.  
  325.         // -- Video functions --
  326.        
  327.         // When only video is playing, delta_time is used to synchronise the video.
  328.         // If video is played together with audio, delta_time is ignored, and video
  329.         // is synchronised based on how many audio frames were consumed.
  330.         // NOTE: the decal's pixel data will change when you call one of "GetVideoFrame" functions again
  331.         olc::Decal* GetVideoFrame(float delta_time);
  332.         // NOTE: the decal's pixel data will change when you call one of "GetVideoFrame" functions again
  333.         olc::Decal* GetVideoFrame();
  334.         MediaResult SkipVideoFrame();
  335.         bool IsVideoOpened();
  336.         // Not all videos have frames of equal length, so FPS can only be average
  337.         double GetAverageVideoFPS();
  338.         void PrintVideoInfo();
  339.        
  340.         //bool seek_frame(int64_t ts);
  341.        
  342.  
  343.         // -- Audio functions --
  344.         // output: pointer to byte array pointed by "void*", where the byte array size must be "channel_count * sample_count * sample_size"
  345.         // Returns amount of samples that were read (if not all samples were written, the rest will be filled with 0s (silence))
  346.         // WARNING: Only use this function if you intend to play the audio yourself. If you do decide to handle audio yourself,
  347.         // note, that when video and audio is played together, video is synchronised according to how many audio samples have been read.
  348.         int GetAudioFrame(void** output, int sample_count);
  349.         bool IsAudioOpened();
  350.         void PrintAudioInfo();
  351.  
  352.     private:
  353.         // -- Video and audio functions --
  354.         MediaResult OpenFile(const char* filename);
  355.         void CloseFile();
  356.         void StartDecodingThread();
  357.         void StopDecodingThread();
  358.         MediaResult DecodingThread();
  359.         static const char* GetError(int errnum);
  360.        
  361.  
  362.         // -- Video functions --
  363.         MediaResult InitVideo();
  364.         void CloseVideo();
  365.         void ConvertFrameToRGBASprite(AVFrame* frame, olc::Sprite* target);
  366.         // Send updated pixel data in olc::Sprite to GPU
  367.         void UpdateResultSprite();
  368.         MediaResult HandleVideoDelay();
  369.         const AVFrame* PeekFrame();
  370.         static AVPixelFormat CorrectDeprecatedPixelFormat(AVPixelFormat pix_fmt);
  371.  
  372.         // -- Audio functions --
  373.         MediaResult InitAudio();
  374.         void CloseAudio();
  375.     };
  376. }
  377.  
  378. // Definitions
  379. namespace olc {
  380.     Media::Media() {
  381.     }
  382.  
  383.     Media::~Media() {
  384.         Close();
  385.     }
  386.  
  387.     MediaResult Media::Open(const char* filename, bool open_video, bool open_audio, uint16_t preloaded_frames_scale) {
  388.         MediaResult result;
  389.  
  390.         OLC_MEDIA_ASSERT(preloaded_frames_scale > 0, MediaResult::Error, "\"preloaded_frames_scale\" can't be 0");
  391.  
  392.         this->preloaded_frames_scale = preloaded_frames_scale;
  393.  
  394.         // If media is already open, close it first
  395.         if (IsVideoOpened() || IsAudioOpened()) {
  396.             Close();
  397.         }
  398.  
  399.         printf("opening file\n");
  400.  
  401.         result = OpenFile(filename);
  402.         if (result != MediaResult::Success) {
  403.             return result;
  404.         }
  405.  
  406.         printf("initing video\n");
  407.  
  408.         if (open_video) {
  409.             result = InitVideo();
  410.             if (result != MediaResult::Success) {
  411.                 return result;
  412.             }
  413.         }
  414.        
  415.         printf("initing audio\n");
  416.  
  417.         if (open_audio) {
  418.             result = InitAudio();
  419.             if (result != MediaResult::Success) {
  420.                 return result;
  421.             }
  422.         }
  423.  
  424.         StartDecodingThread();
  425.  
  426.         result = HandleVideoDelay();
  427.         OLC_MEDIA_ASSERT(result == MediaResult::Success, result, "Couldn't handle video delay");
  428.        
  429.  
  430.         return MediaResult::Success;
  431.     }
  432.  
  433.     void Media::Close() {
  434.         StopDecodingThread();
  435.         printf("Closing file\n");
  436.         CloseFile();
  437.         printf("Closing video\n");
  438.         CloseVideo();
  439.         printf("Closing audio\n");
  440.         CloseAudio();
  441.     }
  442.  
  443.     bool Media::FinishedReading() {
  444.         // If neither of the streams were open, return false
  445.         if (IsVideoOpened() == false && IsAudioOpened() == false)
  446.             return false;
  447.  
  448.         // Check if decoding thread has finished and all the frames were read from the streams that were open
  449.         if (finished_reading) {
  450.             bool video_finished = true;
  451.             bool audio_finished = true;
  452.  
  453.             if (IsVideoOpened() && video_fifo.size() > 0) {
  454.                 video_finished = false;
  455.             }
  456.  
  457.             if (IsAudioOpened() && audio_fifo.size() > 0) {
  458.                 audio_finished = false;
  459.             }
  460.  
  461.             return video_finished && audio_finished;
  462.         }
  463.  
  464.         return false;
  465.     }
  466.  
  467.     olc::Decal* Media::GetVideoFrame(float delta_time) {
  468.         if (IsVideoOpened() == false) {
  469.             printf("Video isn't open\n");
  470.             return nullptr;
  471.         }
  472.  
  473.         if (FinishedReading()) {
  474.             printf("Finished reading video\n");
  475.             return nullptr;
  476.         }
  477.  
  478.         double time_reference;
  479.  
  480.         // If audio is opened, synchronise video with audio
  481.         if (IsAudioOpened()) {
  482.             time_reference = audio_time;
  483.         }
  484.         // Otherwise synchronise it based on how much time has passed between function calls (or allow user to mess with delta time if he wants)
  485.         else {
  486.             delta_time_accumulator += delta_time;
  487.             time_reference = delta_time_accumulator;
  488.         }
  489.  
  490.         // If enough time hasn't passed yet, return the same frame
  491.         if (time_reference < last_video_pts)
  492.             return video_frame.Decal();
  493.  
  494.         while (true) {
  495.             const AVFrame* next_frame = PeekFrame();
  496.  
  497.             // Check if Decoding thread has a next video frame at all
  498.             if (next_frame == nullptr)
  499.                 return video_frame.Decal();
  500.  
  501.             last_video_pts = double(next_frame->best_effort_timestamp * video_time_base.num) / double(video_time_base.den);
  502.  
  503.             // Test Decoding thread later by changing "<=" to ">="
  504.             if (time_reference <= last_video_pts)
  505.                 break;
  506.  
  507.             SkipVideoFrame();
  508.         }
  509.  
  510.         return GetVideoFrame();
  511.     }
  512.  
  513.     olc::Decal* Media::GetVideoFrame() {
  514.         if (IsVideoOpened() == false) {
  515.             printf("Video isn't open\n");
  516.             return nullptr;
  517.         }
  518.  
  519.         if (FinishedReading()) {
  520.             printf("Finished reading video\n");
  521.             return nullptr;
  522.         }
  523.  
  524.         // If decoding thread wasn't quick enough to decode frames return same image.
  525.         // (We don't know if decoding thread isn't quick enough, or if last video frame
  526.         // was decoded, and there are other frames left over, like audio frames)
  527.         if (video_fifo.size() > 0) {
  528.             //printf("v-\n");
  529.  
  530.             AVFrame* frame_ref = video_fifo.front();
  531.  
  532.             ConvertFrameToRGBASprite(frame_ref, video_frame.Sprite());
  533.             UpdateResultSprite();
  534.  
  535.             //printf("vt: %lf\n", double(frame_ref->best_effort_timestamp * video_time_base.num) / double(video_time_base.den));
  536.  
  537.             video_fifo.pop();
  538.  
  539.             conditional.notify_one();
  540.         }
  541.  
  542.         return video_frame.Decal();
  543.     }
  544.  
  545.     // TODO: return error when no more frames are available and there is nothing to skip
  546.     MediaResult Media::SkipVideoFrame() {
  547.         if (IsVideoOpened() == false) {
  548.             printf("Video isn't open\n");
  549.             return MediaResult::Error;
  550.         }
  551.  
  552.         if (FinishedReading()) {
  553.             printf("Finished reading video\n");
  554.             return MediaResult::Error;
  555.         }
  556.  
  557.         // If decoding thread wasn't quick enough to decode frames don't do anything.
  558.         // (We don't know if decoding thread isn't quick enough, or if last video frame
  559.         // was decoded, and there are other frames left over, like audio frames)
  560.         if (video_fifo.size() > 0) {
  561.             video_fifo.pop();
  562.  
  563.             conditional.notify_one();
  564.  
  565.             return MediaResult::Success;
  566.         }
  567.  
  568.         return MediaResult::Error;
  569.     }
  570.  
  571.     bool Media::IsVideoOpened() {
  572.         return video_opened;
  573.     }
  574.  
  575.     double Media::GetAverageVideoFPS() {
  576.         return av_q2d(av_format_ctx->streams[video_stream_index]->avg_frame_rate);
  577.     }
  578.  
  579.     void Media::PrintVideoInfo() {
  580.         if (IsVideoOpened() == false) {
  581.             printf("Video isn't open\n");
  582.             return;
  583.         }
  584.  
  585.         AVStream* video_stream = av_format_ctx->streams[video_stream_index];
  586.         double frame_rate = av_q2d(video_stream->avg_frame_rate);
  587.         int time_base_num = video_stream->time_base.num;
  588.         int time_base_den = video_stream->time_base.den;
  589.         int frame_rate_num = video_stream->avg_frame_rate.num;
  590.         int frame_rate_den = video_stream->avg_frame_rate.den;
  591.  
  592.         // Print time base and fps
  593.         printf("----------------------\n");
  594.         printf("Video info\n");
  595.         printf("Codec: %s\n", av_video_codec->long_name);
  596.         printf("Pixel fmt: %s\n", av_get_pix_fmt_name(av_video_codec_ctx->pix_fmt));
  597.         printf("w: %i   h: %i\n", video_width, video_height);
  598.         printf("Frame rate: %lf\n", frame_rate);
  599.         printf("Time base num: %i\n", time_base_num);
  600.         printf("Time base den: %i\n", time_base_den);
  601.         printf("Frame rate num: %i\n", frame_rate_num);
  602.         printf("Frame rate den: %i\n", frame_rate_den);
  603.         printf("----------------------\n");
  604.     }
  605.  
  606.     int Media::GetAudioFrame(void** output, int sample_count) {
  607.         // Fill buffer with silence, in case not all samples will be read
  608.         memset(*output, 0, audio_channel_count * audio_sample_size * sample_count);
  609.         int samples_read = audio_fifo.pop(output, sample_count);
  610.         conditional.notify_one();
  611.  
  612.         audio_frames_consumed += samples_read;
  613.         audio_time = double(audio_frames_consumed * audio_time_base.num) / double(audio_time_base.den);
  614.         double a_time = audio_time;
  615.         //printf("at: %lf\n", a_time);
  616.  
  617.         //printf("sr: %i\n", samples_read);
  618.  
  619.         //printf("tot: %llu\n", audio_frames_consumed);
  620.         //printf("-----------\n");
  621.  
  622.         // TEMP
  623.         //memset(*output, 0, audio_channel_count * audio_sample_size * sample_count);
  624.  
  625.         return samples_read;
  626.     }
  627.  
  628.     bool Media::IsAudioOpened() {
  629.         return audio_opened;
  630.     }
  631.  
  632.     void Media::PrintAudioInfo() {
  633.         if (IsAudioOpened() == false) {
  634.             printf("Audio isn't open\n");
  635.             return;
  636.         }
  637.  
  638.         AVStream* audio_stream = av_format_ctx->streams[audio_stream_index];
  639.         int frame_size = av_format_ctx->streams[audio_stream_index]->codecpar->frame_size;
  640.         int sample_rate = av_format_ctx->streams[audio_stream_index]->codecpar->sample_rate;
  641.         int channels = av_format_ctx->streams[audio_stream_index]->codecpar->channels;
  642.         int time_base_num = audio_stream->time_base.num;
  643.         int time_base_den = audio_stream->time_base.den;
  644.  
  645.         // Print time base and fps
  646.         printf("----------------------\n");
  647.         printf("Audio info\n");
  648.  
  649.         printf("Codec: %s\n", av_audio_codec->long_name);
  650.         printf("Frame size: %i\n", frame_size);
  651.         printf("Sample rate: %i\n", sample_rate);
  652.         printf("Channels: %i\n", channels);
  653.         printf("Time base num: %i\n", time_base_num);
  654.         printf("Time base den: %i\n", time_base_den);
  655.         printf("block_align: %i\n", av_format_ctx->streams[audio_stream_index]->codecpar->block_align);
  656.         printf("initial_padding: %i\n", av_format_ctx->streams[audio_stream_index]->codecpar->initial_padding);
  657.         printf("trailing_padding: %i\n", av_format_ctx->streams[audio_stream_index]->codecpar->trailing_padding);
  658.         printf("seek_preroll: %i\n", av_format_ctx->streams[audio_stream_index]->codecpar->seek_preroll);
  659.         printf("----------------------\n");
  660.     }
  661.  
  662.     MediaResult Media::OpenFile(const char* filename) {
  663.         int response;
  664.  
  665.         av_format_ctx = avformat_alloc_context();
  666.         OLC_MEDIA_ASSERT(av_format_ctx != nullptr, MediaResult::Error, "Couldn't allocate AVFormatContext");
  667.  
  668.         response = avformat_open_input(&av_format_ctx, filename, NULL, NULL);
  669.         if (response < 0) {
  670.             printf("avformat_open_input response: %s\n", GetError(response));
  671.         }
  672.         OLC_MEDIA_ASSERT(response == 0, MediaResult::Error, "Couldn't find/open file or file format isn't supported");
  673.  
  674.         response = avformat_find_stream_info(av_format_ctx, nullptr);
  675.         OLC_MEDIA_ASSERT(response >= 0, MediaResult::Error, "Couldn't find stream info");
  676.  
  677.         return MediaResult::Success;
  678.     }
  679.  
  680.     void Media::CloseFile() {
  681.         avformat_close_input(&av_format_ctx);
  682.  
  683.         // Don't think this function is really needed, but I put it here for sanity reasons
  684.         avformat_free_context(av_format_ctx);
  685.     }
  686.  
  687.     void Media::StartDecodingThread() {
  688.         printf("Starting thread\n");
  689.         keep_loading = true;
  690.         finished_reading = false;
  691.         this->frame_loader = std::thread(&Media::DecodingThread, this);
  692.     }
  693.  
  694.     void Media::StopDecodingThread() {
  695.         printf("Stopping thread\n");
  696.         keep_loading = false;
  697.         finished_reading = true;
  698.         conditional.notify_one();
  699.         if (frame_loader.joinable()) {
  700.             printf("Thread is joinable\n");
  701.             frame_loader.join();
  702.         }
  703.     }
  704.  
  705.     MediaResult Media::DecodingThread() {
  706.         finished_reading = false;
  707.  
  708.         std::unique_lock<std::mutex> lock(mutex);
  709.  
  710.         // Maximum amount of frames that can be pre-decoded
  711.         size_t max_video_queue_size;
  712.  
  713.         // Minimum amount of frames that should be pre-decoded
  714.         size_t min_video_queue_size;
  715.         size_t min_audio_queue_size;
  716.  
  717.         if (IsVideoOpened()) {
  718.             // "-1" because resizing is disabled, and it allows to avoid overwriting a frame recevied from "GetVideoFrame()"
  719.             max_video_queue_size = video_fifo.capacity() - 1;
  720.             min_video_queue_size = std::max(max_video_queue_size / 2, size_t(1));
  721.  
  722.             printf("max_v: %llu\n", max_video_queue_size);
  723.             printf("min_v: %llu\n", min_video_queue_size);
  724.         }
  725.  
  726.         if (IsAudioOpened()) {
  727.             min_audio_queue_size = std::max(size_t(audio_fifo.capacity() / 2), size_t(1));
  728.  
  729.             printf("min_a: %llu\n", min_audio_queue_size);
  730.         }
  731.  
  732.         int response;
  733.  
  734.         AVFrame* av_audio_frame = av_frame_alloc();
  735.         OLC_MEDIA_ASSERT(av_audio_frame != nullptr, MediaResult::Error, "Couldn't allocate resampled AVFrame");
  736.  
  737.         // Used to store converted "av_audio_frame"
  738.         AVFrame* resampled_audio_frame = av_frame_alloc();
  739.         OLC_MEDIA_ASSERT(resampled_audio_frame != nullptr, MediaResult::Error, "Couldn't allocate resampled AVFrame");
  740.  
  741.         AVPacket* av_packet = av_packet_alloc();
  742.         OLC_MEDIA_ASSERT(av_packet != nullptr, MediaResult::Error, "Couldn't allocate resampled AVFrame");
  743.  
  744.         while (true) {
  745.             while (true) {
  746.                 if (IsVideoOpened() && video_fifo.size() <= min_video_queue_size)
  747.                     break;
  748.  
  749.                 if (IsAudioOpened() && audio_fifo.size() <= min_audio_queue_size)
  750.                     break;
  751.  
  752.                 if (keep_loading == false)
  753.                     break;
  754.  
  755.                 conditional.wait(lock);
  756.             }
  757.  
  758.             if (keep_loading == false)
  759.                 break;
  760.  
  761.  
  762.             // Get remaining audio from previous conversion
  763.             if (IsAudioOpened()) {
  764.                 if (swr_get_delay(swr_audio_resampler, std::max(resampled_audio_frame->sample_rate, av_audio_frame->sample_rate)) > 0) {
  765.                     response = swr_convert_frame(swr_audio_resampler, resampled_audio_frame, nullptr);
  766.                     OLC_MEDIA_ASSERT(response == 0, MediaResult::Error, "Couldn't resample the frame");
  767.  
  768.                     int samples_written = audio_fifo.push((void**)resampled_audio_frame->data, resampled_audio_frame->nb_samples);
  769.  
  770.                     continue;
  771.                 }
  772.             }
  773.  
  774.             // Try reading next packet
  775.             response = av_read_frame(av_format_ctx, av_packet);
  776.  
  777.             // Return if error or end of file was encountered
  778.             if (response < 0) {
  779.                 printf("Error or end of file happened\n");
  780.                 printf("Exit info: %s\n", GetError(response));
  781.  
  782.                 // TODO: check if response is error or end of file
  783.                 break;
  784.             }
  785.  
  786.             if (IsVideoOpened() && av_packet->stream_index == video_stream_index) {
  787.                 PiraTimer::start("DecodeVideoFrame");
  788.  
  789.                 // Drain a frame when max size is reached
  790.                 if (max_video_queue_size == video_fifo.size()) {
  791.                     video_fifo.pop();
  792.                 }
  793.                    
  794.  
  795.                 AVFrame* av_video_frame = video_fifo.back();
  796.  
  797.                 // Send packet to decode
  798.                 response = avcodec_send_packet(av_video_codec_ctx, av_packet);
  799.                 OLC_MEDIA_ASSERT(response == 0, MediaResult::Error, "Couldn't decode packet");
  800.  
  801.                 // Receive decoded frame
  802.                 response = avcodec_receive_frame(av_video_codec_ctx, av_video_frame);
  803.                 if (response < 0) {
  804.                     OLC_MEDIA_ASSERT(response == AVERROR_EOF || response == AVERROR(EAGAIN), MediaResult::Error, "Couldn't receive decoded frame");
  805.                 }
  806.  
  807.                 video_fifo.push();
  808.  
  809.                 PiraTimer::end("DecodeVideoFrame");
  810.             }
  811.             else if (IsAudioOpened() && av_packet->stream_index == audio_stream_index) {
  812.                 PiraTimer::start("DecodeAudioFrame");
  813.  
  814.                 // Send packet to decode
  815.                 response = avcodec_send_packet(av_audio_codec_ctx, av_packet);
  816.                 if (response < 0) {
  817.                     OLC_MEDIA_ASSERT(response == AVERROR(EAGAIN), MediaResult::Error, "Failed to decode packet");
  818.                 }
  819.  
  820.                 // Single packet can contain multiple frames, so receive them in a loop
  821.                 while (true) {
  822.                     response = avcodec_receive_frame(av_audio_codec_ctx, av_audio_frame);
  823.                     if (response < 0) {
  824.                         OLC_MEDIA_ASSERT(response == AVERROR_EOF || response == AVERROR(EAGAIN), MediaResult::Error, "Something went wrong when trying to receive decoded frame");
  825.                         break;
  826.                     }
  827.  
  828.                     // Convert whatever format audio frame is encoded in, to "AV_SAMPLE_FMT_FLT", for simplicity.
  829.                     // We have to manually copy some frame data
  830.                     resampled_audio_frame->sample_rate = av_audio_frame->sample_rate;
  831.                     resampled_audio_frame->channel_layout = av_audio_frame->channel_layout;
  832.                     resampled_audio_frame->channels = av_audio_frame->channels;
  833.                     resampled_audio_frame->format = AV_SAMPLE_FMT_FLT;
  834.  
  835.                     response = swr_convert_frame(swr_audio_resampler, resampled_audio_frame, av_audio_frame);
  836.                     OLC_MEDIA_ASSERT(response == 0, MediaResult::Error, "Couldn't resample the frame");
  837.  
  838.                     av_frame_unref(av_audio_frame);
  839.  
  840.                     // Insert decoded audio samples
  841.                     /*int samples_written = */audio_fifo.push((void**)resampled_audio_frame->data, resampled_audio_frame->nb_samples);
  842.                     //static size_t total_written = 0;
  843.                     //total_written += samples_written;
  844.  
  845.                     //printf("total_written: %llu\n", total_written);
  846.                     //printf("sw: %i\n", samples_written);
  847.                 }
  848.  
  849.                 PiraTimer::end("DecodeAudioFrame");
  850.             }
  851.             //std::this_thread::sleep_for(std::chrono::milliseconds(10));
  852.  
  853.             av_packet_unref(av_packet);
  854.         }
  855.  
  856.         finished_reading = true;
  857.  
  858.         printf("Freeing resources in thread\n");
  859.  
  860.         // Free the resources
  861.         av_frame_free(&av_audio_frame);
  862.         av_frame_free(&resampled_audio_frame);
  863.         av_packet_free(&av_packet);
  864.  
  865.         printf("Exiting thread\n");
  866.  
  867.         return MediaResult::Success;
  868.     }
  869.  
  870.     // av_err2str returns a temporary array. This doesn't work in gcc.
  871.     // This function can be used as a replacement for av_err2str.
  872.     const char* Media::GetError(int errnum) {
  873.         static char str[AV_ERROR_MAX_STRING_SIZE];
  874.         memset(str, 0, sizeof(str));
  875.         return av_make_error_string(str, AV_ERROR_MAX_STRING_SIZE, errnum);
  876.     }
  877.  
  878.     MediaResult Media::InitVideo() {
  879.         int response;
  880.  
  881.         AVCodecParameters* av_video_codec_params = nullptr;
  882.        
  883.         printf("1\n");
  884.  
  885.         video_stream_index = av_find_best_stream(av_format_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &av_video_codec, 0);
  886.         if (video_stream_index < 0) {
  887.             if (video_stream_index == AVERROR_STREAM_NOT_FOUND) {
  888.                 // TODO: might change it later
  889.                 return MediaResult::Success;
  890.             }
  891.             else if (video_stream_index == AVERROR_DECODER_NOT_FOUND) {
  892.                 OLC_MEDIA_ASSERT(false, MediaResult::Error, "Couldn't find decoder for any of the video streams");
  893.             }
  894.             else {
  895.                 OLC_MEDIA_ASSERT(false, MediaResult::Error, "Unknown error occured when trying to find video stream");
  896.             }
  897.         }
  898.  
  899.         av_video_codec_params = av_format_ctx->streams[video_stream_index]->codecpar;
  900.  
  901.         //av_dump_format(av_video_format_ctx, video_stream_index, filename, 0);
  902.         //printf("fps: %f\n", av_q2d(av_video_format_ctx->streams[video_stream_index]->avg_frame_rate));
  903.         //av_video_format_ctx->streams[video_stream_index]->avg_frame_rate;
  904.  
  905.         printf("2\n");
  906.  
  907.         // Set up a codec context for the decoder
  908.         av_video_codec_ctx = avcodec_alloc_context3(av_video_codec);
  909.         OLC_MEDIA_ASSERT(av_video_codec_ctx != nullptr, MediaResult::Error, "Couldn't create AVCodecContext");
  910.  
  911.         printf("3\n");
  912.  
  913.         response = avcodec_parameters_to_context(av_video_codec_ctx, av_video_codec_params);
  914.         OLC_MEDIA_ASSERT(response >= 0, MediaResult::Error, "Couldn't send parameters to AVCodecContext");
  915.  
  916.         printf("4\n");
  917.  
  918.         response = avcodec_open2(av_video_codec_ctx, av_video_codec, NULL);
  919.         OLC_MEDIA_ASSERT(response == 0, MediaResult::Error, "Couldn't initialise AVCodecContext");
  920.  
  921.         printf("5\n");
  922.  
  923.         AVPixelFormat source_pix_fmt = Media::CorrectDeprecatedPixelFormat(av_video_codec_ctx->pix_fmt);
  924.         sws_video_scaler_ctx = sws_getContext(
  925.             av_video_codec_params->width, av_video_codec_params->height, source_pix_fmt,
  926.             av_video_codec_params->width, av_video_codec_params->height, AV_PIX_FMT_RGB0,
  927.             SWS_BILINEAR, NULL, NULL, NULL
  928.         );
  929.         OLC_MEDIA_ASSERT(sws_video_scaler_ctx != nullptr, MediaResult::Error, "Couldn't initialise SwsContext");
  930.  
  931.         printf("6\n");
  932.  
  933.         // Minimum video fifo capacity must stay 2, regardless of video fps
  934.         MediaResult result = video_fifo.init(std::max(uint16_t(preloaded_frames_scale * GetAverageVideoFPS()), uint16_t(2)));
  935.         OLC_MEDIA_ASSERT(result == MediaResult::Success, result, "Couldn't allocate video fifo");
  936.  
  937.         printf("7\n");
  938.  
  939.         video_opened = true;
  940.         printf("01\n");
  941.         video_width = av_video_codec_params->width;
  942.         printf("02\n");
  943.         video_height = av_video_codec_params->height;
  944.         printf("03\n");
  945.         video_frame.Create(video_width, video_height);
  946.         printf("04\n");
  947.         video_time_base = av_format_ctx->streams[video_stream_index]->time_base;
  948.         printf("05\n");
  949.         video_delay = av_video_codec_params->video_delay;
  950.         printf("Video delay: %i\n", video_delay);
  951.  
  952.         // Reset values if video was previously opened
  953.         delta_time_accumulator = 0.0f;
  954.         last_video_pts = 0.0;
  955.  
  956.         PrintVideoInfo();
  957.  
  958.         return MediaResult::Success;
  959.     }
  960.  
  961.     void Media::CloseVideo() {
  962.         avcodec_free_context(&av_video_codec_ctx);
  963.         sws_freeContext(sws_video_scaler_ctx);
  964.         sws_video_scaler_ctx = nullptr;
  965.  
  966.         video_opened = false;
  967.         video_fifo.clear();
  968.         video_fifo.free();
  969.     }
  970.  
  971.     void Media::ConvertFrameToRGBASprite(AVFrame* frame, olc::Sprite* target) {
  972.         // TODO: implement some error checking
  973.  
  974.         // Convert pixel format from (most likely) YUV representation to RGBA
  975.         uint8_t* dest[4] = { (uint8_t*)(target->pColData), NULL, NULL, NULL };
  976.         int dest_linesize[4] = { video_width * 4, 0, 0, 0 };
  977.         sws_scale(sws_video_scaler_ctx, frame->data, frame->linesize, 0, frame->height, dest, dest_linesize);
  978.     }
  979.  
  980.     void Media::UpdateResultSprite() {
  981.         video_frame.Decal()->Update();
  982.     }
  983.  
  984.     MediaResult Media::HandleVideoDelay() {
  985.         if (IsVideoOpened()) {
  986.             // Skip initial frames provided in "video_delay" after starting the decoding thread
  987.             for (int frames_skipped = 0; frames_skipped < video_delay;) {
  988.                 if (SkipVideoFrame() == MediaResult::Success) {
  989.                     ++frames_skipped;
  990.                 }
  991.                 else {
  992.                     // If video delay is bigger than the video frame count, it probably means video is corrupted or something went wrong
  993.                     if (FinishedReading()) {
  994.                         return MediaResult::Error;
  995.                     }
  996.  
  997.                     // Otherwise thread didn't decode the frames yet, so let's wait a bit
  998.                     std::this_thread::sleep_for(std::chrono::milliseconds(5));
  999.                 }
  1000.             }
  1001.         }
  1002.  
  1003.         return MediaResult::Success;
  1004.     }
  1005.  
  1006.     const AVFrame* Media::PeekFrame() {
  1007.         if (video_fifo.size() > 0) {
  1008.             return video_fifo.front();
  1009.         }
  1010.  
  1011.         return nullptr;
  1012.     }
  1013.  
  1014.     AVPixelFormat Media::CorrectDeprecatedPixelFormat(AVPixelFormat pix_fmt) {
  1015.         // Fix swscaler deprecated pixel format warning
  1016.         // (YUVJ has been deprecated, change pixel format to regular YUV)
  1017.         switch (pix_fmt) {
  1018.         case AV_PIX_FMT_YUVJ420P: return AV_PIX_FMT_YUV420P;
  1019.         case AV_PIX_FMT_YUVJ422P: return AV_PIX_FMT_YUV422P;
  1020.         case AV_PIX_FMT_YUVJ444P: return AV_PIX_FMT_YUV444P;
  1021.         case AV_PIX_FMT_YUVJ440P: return AV_PIX_FMT_YUV440P;
  1022.         default:                  return pix_fmt;
  1023.         }
  1024.     }
  1025.  
  1026.     // TODO: figure out what to do with these variables in AVCodecParameters:
  1027.     // - block_align
  1028.     // - initial_padding
  1029.     // - trailing_padding
  1030.     // - seek_preroll
  1031.     MediaResult Media::InitAudio() {
  1032.         int response;
  1033.  
  1034.         AVCodecParameters* av_audio_codec_params = nullptr;
  1035.  
  1036.         audio_stream_index = av_find_best_stream(av_format_ctx, AVMEDIA_TYPE_AUDIO, -1, -1, &av_audio_codec, 0);
  1037.         if (audio_stream_index < 0) {
  1038.             if (audio_stream_index == AVERROR_STREAM_NOT_FOUND) {
  1039.                 // TODO: might change it later
  1040.                 return MediaResult::Success;
  1041.             }  
  1042.             else if (audio_stream_index == AVERROR_DECODER_NOT_FOUND) {
  1043.                 OLC_MEDIA_ASSERT(false, MediaResult::Error, "Couldn't find decoder for any of the audio streams");
  1044.             }  
  1045.             else {
  1046.                 OLC_MEDIA_ASSERT(false, MediaResult::Error, "Unknown error occured when trying to find audio stream");
  1047.             }
  1048.         }
  1049.  
  1050.         av_audio_codec_params = av_format_ctx->streams[audio_stream_index]->codecpar;
  1051.  
  1052.         // Set up a codec context for the decoder
  1053.         av_audio_codec_ctx = avcodec_alloc_context3(av_audio_codec);
  1054.         OLC_MEDIA_ASSERT(av_audio_codec_ctx != nullptr, MediaResult::Error, "Couldn't create AVCodecContext");
  1055.  
  1056.         response = avcodec_parameters_to_context(av_audio_codec_ctx, av_audio_codec_params);
  1057.         OLC_MEDIA_ASSERT(response >= 0, MediaResult::Error, "Couldn't send parameters to AVCodecContext");
  1058.  
  1059.         response = avcodec_open2(av_audio_codec_ctx, av_audio_codec, NULL);
  1060.         OLC_MEDIA_ASSERT(response == 0, MediaResult::Error, "Couldn't initialise AVCodecContext");
  1061.        
  1062.         swr_audio_resampler = swr_alloc_set_opts(
  1063.             nullptr,
  1064.             av_audio_codec_params->channel_layout, AV_SAMPLE_FMT_FLT, av_audio_codec_params->sample_rate,
  1065.             av_audio_codec_params->channel_layout, (AVSampleFormat)av_audio_codec_params->format, av_audio_codec_params->sample_rate,
  1066.             0, nullptr
  1067.         );
  1068.         OLC_MEDIA_ASSERT(swr_audio_resampler != nullptr, MediaResult::Error, "Couldn't allocate SwrContext");
  1069.  
  1070.         audio_opened = true;
  1071.         audio_sample_size = 4;
  1072.         audio_channel_count = av_audio_codec_params->channels;
  1073.         audio_time_base = av_format_ctx->streams[audio_stream_index]->time_base;
  1074.  
  1075.         // Reset values if audio was previously opened
  1076.         audio_frames_consumed = 0;
  1077.         audio_time = 0.0;
  1078.  
  1079.         MediaResult result = audio_fifo.init(av_audio_codec_params->channels, preloaded_frames_scale * av_audio_codec_params->sample_rate);
  1080.         OLC_MEDIA_ASSERT(result == MediaResult::Success, result, "Couldn't allocate audio fifo");
  1081.  
  1082.  
  1083.         // Initialise and immediately start audio device
  1084.         ma_device_config audio_device_config;
  1085.  
  1086.         audio_device_config = ma_device_config_init(ma_device_type_playback);
  1087.         audio_device_config.playback.format = ma_format_f32;
  1088.         audio_device_config.playback.channels = av_audio_codec_params->channels;
  1089.         audio_device_config.sampleRate = av_audio_codec_params->sample_rate;
  1090.         audio_device_config.pUserData = this;
  1091.         audio_device_config.dataCallback = [](ma_device* pDevice, void* pOutput, const void* pInput, ma_uint32 frameCount) {
  1092.             Media* media = reinterpret_cast<Media*>(pDevice->pUserData);
  1093.             int frames_read = media->GetAudioFrame(&pOutput, frameCount);
  1094.  
  1095.             //std::this_thread::sleep_for(std::chrono::milliseconds(200));
  1096.             (void)pInput;
  1097.         };
  1098.  
  1099.         OLC_MEDIA_ASSERT(ma_device_init(NULL, &audio_device_config, &audio_device) == MA_SUCCESS, MediaResult::Error, "Couldn't open playback device");
  1100.  
  1101.         OLC_MEDIA_ASSERT(ma_device_start(&audio_device) == MA_SUCCESS, MediaResult::Error, "Couldn't start playback device");
  1102.  
  1103.         // TODO: figure out if it's possible to neatly put this into assert macro
  1104.         /*if (ma_device_start(&audio_device) != MA_SUCCESS) {
  1105.             printf("Failed to start playback device.\n");
  1106.             ma_device_uninit(&audio_device);
  1107.             return MediaResult::Error;
  1108.         }*/
  1109.  
  1110.         PrintAudioInfo();
  1111.  
  1112.         return MediaResult::Success;
  1113.     }
  1114.  
  1115.     void Media::CloseAudio() {
  1116.         avcodec_free_context(&av_audio_codec_ctx);
  1117.         swr_free(&swr_audio_resampler);
  1118.         ma_device_uninit(&audio_device);
  1119.  
  1120.         audio_opened = false;
  1121.         video_fifo.clear();
  1122.         video_fifo.free();
  1123.     }
  1124.    
  1125.     /*bool Media2::seek_frame(int64_t ts) {
  1126.         av_seek_frame(av_video_format_ctx, video_stream_index, ts, AVSEEK_FLAG_BACKWARD);
  1127.  
  1128.         // av_seek_frame takes effect after one frame, so I'm decoding one here
  1129.         // so that the next call to video_reader_read_frame() will give the correct
  1130.         // frame
  1131.         int response;
  1132.         while (av_read_frame(av_video_format_ctx, av_video_packet) >= 0) {
  1133.             if (av_video_packet->stream_index != video_stream_index) {
  1134.                 av_packet_unref(av_packet);
  1135.                 continue;
  1136.             }
  1137.  
  1138.             response = avcodec_send_packet(av_codec_ctx, av_packet);
  1139.             if (response < 0) {
  1140.                 printf("Failed to decode packet: %s\n", av_make_error(response));
  1141.                 return false;
  1142.             }
  1143.  
  1144.             response = avcodec_receive_frame(av_codec_ctx, av_frame);
  1145.             if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
  1146.                 av_packet_unref(av_packet);
  1147.                 continue;
  1148.             }
  1149.             else if (response < 0) {
  1150.                 printf("Failed to decode packet: %s\n", av_make_error(response));
  1151.                 return false;
  1152.             }
  1153.  
  1154.             av_packet_unref(av_packet);
  1155.             break;
  1156.         }
  1157.  
  1158.         return true;
  1159.     }*/
  1160. }
  1161.  
  1162. #endif // OLCPGEX_MEDIA_H
Advertisement
Add Comment
Please, Sign In to add comment