Ffmpeg+sdl2 for audio playback

This document records the use of ffmpeg+SDL2 Play the audio in the video file , Note: play the audio in the video file , Not playing audio files .

This article uses ffmpeg Version is 5.0.1,SDL The version is 2.022.c++ The environment is vs2017.

The obvious difference from playing video or audio before is , Resampling is required to play the audio in the video file , Resampling structures are introduced in the code SwrContext.

The resampling structure can change the sampling rate of the original audio 、 Number of channels and other parameters , So that all kinds of audio can be output according to the parameters we set . The reason for this is that the audio parameters in different video files usually differ greatly , If the separate processing workload is too large , It is better to unify them into the same format .

First overall code ：

#include <stdio.h>
#include <tchar.h>
#include <iostream>


extern "C"
{
#include "SDL.h" // The header file should not only be configured by clicking the mouse in the project , We should also introduce... Into the code 
#include "include/libavformat/avformat.h"
#include "include/libavformat/avformat.h"
#include "include/libswscale/swscale.h"
#include "include/libavdevice/avdevice.h"
#include "include/libavcodec/avcodec.h"
#include "include/libswresample/swresample.h"
};
using namespace std;

#define MAX_AUDIO_FRAME_SIZE 19200

static  Uint8  *audio_chunk;  // Audio block 
static  Uint32  audio_len;    // The remaining length of the audio 
static  Uint8  *audio_pos;    // The current location of the audio 

// The callback function is used to fill the audio buffer , This callback function will be called when the audio device needs more data 
//userdata Generally not used ,stream It's an audio buffer ,len Is the size of the audio buffer 
void  fill_audio(void *udata, Uint8 *stream, int len) {
	// take stream Set up 0,SDL2 Necessary operation 
	SDL_memset(stream, 0, len);
	if (audio_len == 0)		// If there is no data left 
		return;
	len = (len > audio_len ? audio_len : len);	// Get as much data as possible 
	// Mix processing 
	SDL_MixAudio(stream, audio_pos, len, SDL_MIX_MAXVOLUME);
	audio_pos += len;   // Update the current location of the audio 
	audio_len -= len;   // Update the remaining length of the audio 
}

#undef main
int main(int argc, char* argv[])
{
	const char *audio_path = "ds.mov";// Video path 

	// initialization ffmpeg Components 
	AVFormatContext *pFormatCtx = nullptr;
	int i, audioStream = -1,videoStream = -1;
	AVCodecParameters *pCodecParameters = nullptr;
	AVCodecContext *pCodecCtx = nullptr;
	const AVCodec *pCodec = nullptr;
	AVFrame *pFrame = nullptr;
	AVPacket *packet;
	uint8_t *out_buffer;

	int64_t in_channel_layout;

	// Initialize the resample component 
	struct SwrContext *au_convert_ctx;

	// Open the audio file  ffmpeg
	if (avformat_open_input(&pFormatCtx, audio_path, nullptr, nullptr) != 0) {
		cout << "can not open the audio!" << endl;
		return -1;
	}

	// Find video and audio streams  ffmpeg
	audioStream = av_find_best_stream(pFormatCtx, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0);
	if (audioStream == -1) {
		cout << "can not find audio stream!" << endl;
		return -1;
	}
	videoStream = av_find_best_stream(pFormatCtx, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
	if (videoStream == -1) {
		cout << "can not open a video stream" << endl;
		return -1;
	}

	// Looking for decoder  ffmpeg
	pCodecParameters = pFormatCtx->streams[audioStream]->codecpar;
	pCodec = avcodec_find_decoder(pFormatCtx->streams[audioStream]->codecpar->codec_id);
	if (pCodec == nullptr) {
		cout << "can not find a codec" << endl;
		return -1;
	}

	// Load decoder parameters  ffmpeg
	pCodecCtx = avcodec_alloc_context3(pCodec);
	if (avcodec_parameters_to_context(pCodecCtx, pCodecParameters) != 0) {
		cout << "can not copy codec context" << endl;
		return -1;
	}

	// Start decoder  ffmpeg
	if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0) {
		cout << "can not open the decoder!" << endl;
		return -1;
	}

	// initialization packet And pframe ffmpeg
	packet = (AVPacket*)av_malloc(sizeof(AVPacket));
	av_packet_alloc();
	pFrame = av_frame_alloc();

	// Audio parameter initialization 
	uint64_t out_channel_layout = AV_CH_LAYOUT_STEREO;// Output channel 
	int out_nb_samples = 1024; // The number of samples in the audio buffer 
	enum AVSampleFormat out_sample_fmt = AV_SAMPLE_FMT_S16;// Output format S16
	int out_sample_rate = 44100; //pcm Sampling rate 
	int out_channels = av_get_channel_layout_nb_channels(out_channel_layout);// Number of channels 

	int out_buffer_size = av_samples_get_buffer_size(NULL, out_channels, out_nb_samples, out_sample_fmt, 1);// Calculate audio buffer size 
	out_buffer = (uint8_t *)av_malloc(MAX_AUDIO_FRAME_SIZE * 2);// Initialize audio buffer size 

	// initialization SDL
	if (SDL_Init(SDL_INIT_AUDIO | SDL_INIT_TIMER)) {
		cout << "can not open the SDL!" << endl;
		return -1;
	}

	// Set audio related parameters 
	SDL_AudioSpec spec;
	spec.freq = out_sample_rate;  //pcm sampling frequency 
	spec.format = AUDIO_S16SYS;   // Audio format ,16bit
	spec.channels = out_channels; // Number of channels 
	spec.silence = 0;             // Set the mute value    
	spec.samples = out_nb_samples;// The number of samples in the audio buffer 
	spec.callback = fill_audio;   // Callback function 
	spec.userdata = pCodecCtx;    // The data of the callback function comes from the decoder 

	// initialization SDL
	if (SDL_OpenAudio(&spec, nullptr) < 0) {
		cout << "can not open audio" << endl;
		return -1;
	}

	// Audio resampling operation 
	in_channel_layout = av_get_default_channel_layout(pCodecCtx->channels);
	cout << "in_channel_layout: " << in_channel_layout << endl;
	au_convert_ctx = swr_alloc(); // Initialize the resample structure 
	au_convert_ctx = swr_alloc_set_opts(au_convert_ctx, out_channel_layout, out_sample_fmt, out_sample_rate, in_channel_layout, pCodecCtx->sample_fmt, pCodecCtx->sample_rate, 0, NULL);// Resample structure assignment 
	swr_init(au_convert_ctx);// Load the resampled structure parameters 

	SDL_PauseAudio(0);

	// Cyclic reading packet
	while (av_read_frame(pFormatCtx, packet) >= 0) {
		if (packet->stream_index == audioStream) {
			// To decode 
			avcodec_send_packet(pCodecCtx, packet);
			while (avcodec_receive_frame(pCodecCtx,pFrame) == 0)
			{
				// Convert the input audio according to the defined parameters , And the output 
				swr_convert(au_convert_ctx, &out_buffer, MAX_AUDIO_FRAME_SIZE, (const uint8_t **)pFrame->data, pFrame->nb_samples);
			}

			audio_chunk = (Uint8*)out_buffer; // Set the audio buffer 
			audio_len = out_buffer_size;      // Update audio length 
			audio_pos = audio_chunk;          // Update audio location 
			while (audio_len > 0)
			{
				SDL_Delay(1);// Delay playback 
			}
		}
		av_packet_unref(packet);// Empty packet Content 
	}
	swr_free(&au_convert_ctx);// Clear the resample structure contents 
	// Recycling ffmpeg Components 
	if (pFrame) {
		av_frame_free(&pFrame);
		pFrame = nullptr;
	}

	if (pCodecCtx) {
		avcodec_close(pCodecCtx);
		pCodecCtx = nullptr;
		pCodec = nullptr;
	}

	if (pFormatCtx) {
		avformat_close_input(&pFormatCtx);
		pFormatCtx = nullptr;
	}
	// close SDL
	SDL_Quit();
	cout << "succeed!" << endl;
	return 0;
}

You can see that in the code , except ffmpeg And sdl Relevant old content , Added... To the code swr Resampled contents of .

    // Initialize the resample component 
	struct SwrContext *au_convert_ctx;

First, initialize the resampling component , The structure can record various audio parameters after resampling .

    au_convert_ctx = swr_alloc(); // Initialize the resample structure 
	au_convert_ctx = swr_alloc_set_opts(au_convert_ctx, out_channel_layout, out_sample_fmt, out_sample_rate, in_channel_layout, pCodecCtx->sample_fmt, pCodecCtx->sample_rate, 0, NULL);// Resample structure assignment 
	swr_init(au_convert_ctx);// Load the resampled structure parameters 

When setting parameters for the structure , The format and sampling rate should be consistent with the audio in the original video , Therefore, it is necessary to set the parameters of the resampling component after starting the decoder .

// Convert the input audio according to the defined parameters , And the output 
swr_convert(au_convert_ctx, &out_buffer, MAX_AUDIO_FRAME_SIZE, (const uint8_t **)pFrame->data, pFrame->nb_samples);

In a loop operation , Use the above functions to resample, convert and output the audio , It can be seen that it also simplifies the code content required for output .