Advanced Linux Audio and Multimedia Programming: Real-Time Audio Processing and Media Framework Development
Linux multimedia programming requires deep understanding of audio subsystems, real-time processing constraints, and multimedia framework architectures. This comprehensive guide explores advanced audio programming techniques, from low-level ALSA development to building complete multimedia processing pipelines with FFmpeg and custom DSP implementations.
Advanced Linux Audio and Multimedia Programming
ALSA Advanced Programming and Real-Time Audio
Low-Level ALSA PCM Programming Framework
// alsa_advanced.c - Advanced ALSA programming framework
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <math.h>
#include <time.h>
#include <signal.h>
#include <pthread.h>
#include <sys/time.h>
#include <alsa/asoundlib.h>
#include <fftw3.h>
#include <samplerate.h>
#include <sndfile.h>
#define SAMPLE_RATE 48000
#define CHANNELS 2
#define PERIOD_SIZE 256
#define BUFFER_SIZE (PERIOD_SIZE * 4)
#define FORMAT SND_PCM_FORMAT_S32_LE
#define MAX_LATENCY_MS 10
#define RT_PRIORITY 95
// Audio processing context
typedef struct {
snd_pcm_t *playback_handle;
snd_pcm_t *capture_handle;
snd_pcm_hw_params_t *hw_params;
snd_pcm_sw_params_t *sw_params;
unsigned int sample_rate;
unsigned int channels;
snd_pcm_uframes_t period_size;
snd_pcm_uframes_t buffer_size;
snd_pcm_format_t format;
// Real-time processing
pthread_t audio_thread;
bool running;
int priority;
// Buffers
int32_t *input_buffer;
int32_t *output_buffer;
float *float_buffer;
// Performance monitoring
struct {
unsigned long xruns;
unsigned long underruns;
unsigned long overruns;
double avg_latency_ms;
double max_latency_ms;
unsigned long processed_frames;
} stats;
// DSP processing chain
void (*process_callback)(struct audio_context *ctx, float *input, float *output,
snd_pcm_uframes_t frames);
void *user_data;
} audio_context_t;
// DSP processing structures
typedef struct {
float *delay_line;
size_t delay_samples;
size_t write_pos;
float feedback;
float wet_level;
} delay_effect_t;
typedef struct {
float cutoff;
float resonance;
float a0, a1, a2, b1, b2;
float x1, x2, y1, y2;
} biquad_filter_t;
typedef struct {
fftw_complex *input;
fftw_complex *output;
fftw_plan forward_plan;
fftw_plan inverse_plan;
size_t fft_size;
float *window;
float *overlap_buffer;
size_t overlap_size;
} spectral_processor_t;
// Global context
static audio_context_t *g_audio_ctx = NULL;
static volatile bool g_shutdown = false;
// Utility functions
static void set_realtime_priority(int priority) {
struct sched_param param;
param.sched_priority = priority;
if (sched_setscheduler(0, SCHED_FIFO, ¶m) != 0) {
perror("sched_setscheduler");
printf("Warning: Could not set real-time priority. Run as root for RT scheduling.\n");
} else {
printf("Set real-time priority to %d\n", priority);
}
}
static void lock_memory(void) {
if (mlockall(MCL_CURRENT | MCL_FUTURE) != 0) {
perror("mlockall");
printf("Warning: Could not lock memory pages\n");
}
}
// ALSA device setup and configuration
static int setup_alsa_device(audio_context_t *ctx, const char *device_name,
snd_pcm_stream_t stream, snd_pcm_t **handle) {
int err;
snd_pcm_hw_params_t *hw_params;
snd_pcm_sw_params_t *sw_params;
// Open PCM device
err = snd_pcm_open(handle, device_name, stream, SND_PCM_NONBLOCK);
if (err < 0) {
fprintf(stderr, "Cannot open %s PCM device %s: %s\n",
snd_pcm_stream_name(stream), device_name, snd_strerror(err));
return err;
}
// Allocate hardware parameters
snd_pcm_hw_params_alloca(&hw_params);
err = snd_pcm_hw_params_any(*handle, hw_params);
if (err < 0) {
fprintf(stderr, "Cannot initialize hardware parameter structure: %s\n",
snd_strerror(err));
return err;
}
// Set access mode
err = snd_pcm_hw_params_set_access(*handle, hw_params, SND_PCM_ACCESS_RW_INTERLEAVED);
if (err < 0) {
fprintf(stderr, "Cannot set access type: %s\n", snd_strerror(err));
return err;
}
// Set sample format
err = snd_pcm_hw_params_set_format(*handle, hw_params, ctx->format);
if (err < 0) {
fprintf(stderr, "Cannot set sample format: %s\n", snd_strerror(err));
return err;
}
// Set sample rate
unsigned int rate = ctx->sample_rate;
err = snd_pcm_hw_params_set_rate_near(*handle, hw_params, &rate, 0);
if (err < 0) {
fprintf(stderr, "Cannot set sample rate: %s\n", snd_strerror(err));
return err;
}
if (rate != ctx->sample_rate) {
printf("Rate doesn't match (requested %uHz, got %uHz)\n", ctx->sample_rate, rate);
ctx->sample_rate = rate;
}
// Set number of channels
err = snd_pcm_hw_params_set_channels(*handle, hw_params, ctx->channels);
if (err < 0) {
fprintf(stderr, "Cannot set channel count: %s\n", snd_strerror(err));
return err;
}
// Set period size
snd_pcm_uframes_t period_size = ctx->period_size;
err = snd_pcm_hw_params_set_period_size_near(*handle, hw_params, &period_size, 0);
if (err < 0) {
fprintf(stderr, "Cannot set period size: %s\n", snd_strerror(err));
return err;
}
ctx->period_size = period_size;
// Set buffer size
snd_pcm_uframes_t buffer_size = ctx->buffer_size;
err = snd_pcm_hw_params_set_buffer_size_near(*handle, hw_params, &buffer_size);
if (err < 0) {
fprintf(stderr, "Cannot set buffer size: %s\n", snd_strerror(err));
return err;
}
ctx->buffer_size = buffer_size;
// Apply hardware parameters
err = snd_pcm_hw_params(*handle, hw_params);
if (err < 0) {
fprintf(stderr, "Cannot set hardware parameters: %s\n", snd_strerror(err));
return err;
}
// Configure software parameters
snd_pcm_sw_params_alloca(&sw_params);
err = snd_pcm_sw_params_current(*handle, sw_params);
if (err < 0) {
fprintf(stderr, "Cannot get software parameters: %s\n", snd_strerror(err));
return err;
}
// Set start threshold
err = snd_pcm_sw_params_set_start_threshold(*handle, sw_params, period_size);
if (err < 0) {
fprintf(stderr, "Cannot set start threshold: %s\n", snd_strerror(err));
return err;
}
// Set stop threshold
err = snd_pcm_sw_params_set_stop_threshold(*handle, sw_params, buffer_size);
if (err < 0) {
fprintf(stderr, "Cannot set stop threshold: %s\n", snd_strerror(err));
return err;
}
// Apply software parameters
err = snd_pcm_sw_params(*handle, sw_params);
if (err < 0) {
fprintf(stderr, "Cannot set software parameters: %s\n", snd_strerror(err));
return err;
}
printf("ALSA %s device configured:\n", snd_pcm_stream_name(stream));
printf(" Sample rate: %u Hz\n", ctx->sample_rate);
printf(" Channels: %u\n", ctx->channels);
printf(" Period size: %lu frames\n", ctx->period_size);
printf(" Buffer size: %lu frames\n", ctx->buffer_size);
printf(" Latency: %.2f ms\n",
(double)ctx->period_size / ctx->sample_rate * 1000.0);
return 0;
}
// DSP Processing Functions
// Biquad filter implementation
static void biquad_filter_init(biquad_filter_t *filter, float cutoff, float resonance,
float sample_rate) {
filter->cutoff = cutoff;
filter->resonance = resonance;
// Calculate filter coefficients (lowpass)
float omega = 2.0f * M_PI * cutoff / sample_rate;
float sin_omega = sinf(omega);
float cos_omega = cosf(omega);
float alpha = sin_omega / (2.0f * resonance);
float a0 = 1.0f + alpha;
filter->a0 = (1.0f - cos_omega) / (2.0f * a0);
filter->a1 = (1.0f - cos_omega) / a0;
filter->a2 = (1.0f - cos_omega) / (2.0f * a0);
filter->b1 = -2.0f * cos_omega / a0;
filter->b2 = (1.0f - alpha) / a0;
filter->x1 = filter->x2 = filter->y1 = filter->y2 = 0.0f;
}
static float biquad_filter_process(biquad_filter_t *filter, float input) {
float output = filter->a0 * input + filter->a1 * filter->x1 + filter->a2 * filter->x2
- filter->b1 * filter->y1 - filter->b2 * filter->y2;
filter->x2 = filter->x1;
filter->x1 = input;
filter->y2 = filter->y1;
filter->y1 = output;
return output;
}
// Delay effect implementation
static delay_effect_t* delay_effect_create(float delay_ms, float sample_rate,
float feedback, float wet_level) {
delay_effect_t *delay = malloc(sizeof(delay_effect_t));
if (!delay) return NULL;
delay->delay_samples = (size_t)(delay_ms * sample_rate / 1000.0f);
delay->delay_line = calloc(delay->delay_samples, sizeof(float));
if (!delay->delay_line) {
free(delay);
return NULL;
}
delay->write_pos = 0;
delay->feedback = feedback;
delay->wet_level = wet_level;
return delay;
}
static float delay_effect_process(delay_effect_t *delay, float input) {
float delayed = delay->delay_line[delay->write_pos];
delay->delay_line[delay->write_pos] = input + delayed * delay->feedback;
delay->write_pos = (delay->write_pos + 1) % delay->delay_samples;
return input + delayed * delay->wet_level;
}
static void delay_effect_destroy(delay_effect_t *delay) {
if (delay) {
free(delay->delay_line);
free(delay);
}
}
// Spectral processing framework
static spectral_processor_t* spectral_processor_create(size_t fft_size) {
spectral_processor_t *proc = malloc(sizeof(spectral_processor_t));
if (!proc) return NULL;
proc->fft_size = fft_size;
proc->overlap_size = fft_size / 2;
// Allocate FFT buffers
proc->input = fftw_malloc(sizeof(fftw_complex) * fft_size);
proc->output = fftw_malloc(sizeof(fftw_complex) * fft_size);
proc->overlap_buffer = calloc(proc->overlap_size, sizeof(float));
if (!proc->input || !proc->output || !proc->overlap_buffer) {
spectral_processor_destroy(proc);
return NULL;
}
// Create FFT plans
proc->forward_plan = fftw_plan_dft_1d(fft_size, proc->input, proc->output,
FFTW_FORWARD, FFTW_ESTIMATE);
proc->inverse_plan = fftw_plan_dft_1d(fft_size, proc->output, proc->input,
FFTW_BACKWARD, FFTW_ESTIMATE);
// Create Hann window
proc->window = malloc(sizeof(float) * fft_size);
for (size_t i = 0; i < fft_size; i++) {
proc->window[i] = 0.5f * (1.0f - cosf(2.0f * M_PI * i / (fft_size - 1)));
}
return proc;
}
static void spectral_processor_destroy(spectral_processor_t *proc) {
if (proc) {
if (proc->input) fftw_free(proc->input);
if (proc->output) fftw_free(proc->output);
if (proc->overlap_buffer) free(proc->overlap_buffer);
if (proc->window) free(proc->window);
if (proc->forward_plan) fftw_destroy_plan(proc->forward_plan);
if (proc->inverse_plan) fftw_destroy_plan(proc->inverse_plan);
free(proc);
}
}
// Example spectral processing function (noise reduction)
static void spectral_noise_reduction(spectral_processor_t *proc, float *audio_data,
size_t frames, float noise_floor) {
for (size_t i = 0; i < frames; i += proc->overlap_size) {
size_t block_size = (i + proc->fft_size <= frames) ? proc->fft_size : frames - i;
// Apply window and copy to FFT input
for (size_t j = 0; j < block_size; j++) {
proc->input[j][0] = audio_data[i + j] * proc->window[j];
proc->input[j][1] = 0.0f;
}
// Pad with zeros if necessary
for (size_t j = block_size; j < proc->fft_size; j++) {
proc->input[j][0] = 0.0f;
proc->input[j][1] = 0.0f;
}
// Forward FFT
fftw_execute(proc->forward_plan);
// Spectral processing (noise reduction)
for (size_t j = 0; j < proc->fft_size; j++) {
float magnitude = sqrtf(proc->output[j][0] * proc->output[j][0] +
proc->output[j][1] * proc->output[j][1]);
if (magnitude < noise_floor) {
// Suppress noise
float suppression = magnitude / noise_floor;
proc->output[j][0] *= suppression;
proc->output[j][1] *= suppression;
}
}
// Inverse FFT
fftw_execute(proc->inverse_plan);
// Overlap-add reconstruction
for (size_t j = 0; j < proc->overlap_size && i + j < frames; j++) {
audio_data[i + j] = (proc->input[j][0] / proc->fft_size +
proc->overlap_buffer[j]) * proc->window[j];
}
// Store overlap for next block
for (size_t j = 0; j < proc->overlap_size && j + proc->overlap_size < proc->fft_size; j++) {
proc->overlap_buffer[j] = proc->input[j + proc->overlap_size][0] / proc->fft_size;
}
}
}
// Default audio processing callback
static void default_process_callback(audio_context_t *ctx, float *input, float *output,
snd_pcm_uframes_t frames) {
// Simple passthrough with gain
float gain = 0.8f;
for (snd_pcm_uframes_t i = 0; i < frames * ctx->channels; i++) {
output[i] = input[i] * gain;
}
}
// Audio processing thread
static void* audio_thread_func(void *arg) {
audio_context_t *ctx = (audio_context_t*)arg;
struct timespec start_time, end_time;
snd_pcm_sframes_t frames_read, frames_written;
printf("Audio processing thread started\n");
// Set thread priority
set_realtime_priority(ctx->priority);
while (ctx->running && !g_shutdown) {
clock_gettime(CLOCK_MONOTONIC, &start_time);
// Read audio input
frames_read = snd_pcm_readi(ctx->capture_handle, ctx->input_buffer, ctx->period_size);
if (frames_read < 0) {
if (frames_read == -EPIPE) {
printf("Input overrun occurred\n");
ctx->stats.overruns++;
snd_pcm_prepare(ctx->capture_handle);
continue;
} else if (frames_read == -EAGAIN) {
continue;
} else {
fprintf(stderr, "Read error: %s\n", snd_strerror(frames_read));
break;
}
}
if (frames_read != ctx->period_size) {
printf("Short read: %ld frames\n", frames_read);
}
// Convert to float for processing
for (snd_pcm_uframes_t i = 0; i < frames_read * ctx->channels; i++) {
ctx->float_buffer[i] = (float)ctx->input_buffer[i] / INT32_MAX;
}
// Apply DSP processing
if (ctx->process_callback) {
ctx->process_callback(ctx, ctx->float_buffer, ctx->float_buffer, frames_read);
}
// Convert back to integer
for (snd_pcm_uframes_t i = 0; i < frames_read * ctx->channels; i++) {
float sample = ctx->float_buffer[i] * INT32_MAX;
if (sample > INT32_MAX) sample = INT32_MAX;
if (sample < INT32_MIN) sample = INT32_MIN;
ctx->output_buffer[i] = (int32_t)sample;
}
// Write audio output
frames_written = snd_pcm_writei(ctx->playback_handle, ctx->output_buffer, frames_read);
if (frames_written < 0) {
if (frames_written == -EPIPE) {
printf("Output underrun occurred\n");
ctx->stats.underruns++;
snd_pcm_prepare(ctx->playback_handle);
continue;
} else if (frames_written == -EAGAIN) {
continue;
} else {
fprintf(stderr, "Write error: %s\n", snd_strerror(frames_written));
break;
}
}
// Update statistics
ctx->stats.processed_frames += frames_read;
clock_gettime(CLOCK_MONOTONIC, &end_time);
double latency_ms = (end_time.tv_sec - start_time.tv_sec) * 1000.0 +
(end_time.tv_nsec - start_time.tv_nsec) / 1000000.0;
ctx->stats.avg_latency_ms = (ctx->stats.avg_latency_ms * 0.99) + (latency_ms * 0.01);
if (latency_ms > ctx->stats.max_latency_ms) {
ctx->stats.max_latency_ms = latency_ms;
}
// Check for excessive latency
if (latency_ms > MAX_LATENCY_MS) {
printf("Warning: High processing latency: %.2f ms\n", latency_ms);
}
}
printf("Audio processing thread finished\n");
return NULL;
}
// Initialize audio context
static audio_context_t* audio_context_create(const char *playback_device,
const char *capture_device) {
audio_context_t *ctx = calloc(1, sizeof(audio_context_t));
if (!ctx) return NULL;
// Set default parameters
ctx->sample_rate = SAMPLE_RATE;
ctx->channels = CHANNELS;
ctx->period_size = PERIOD_SIZE;
ctx->buffer_size = BUFFER_SIZE;
ctx->format = FORMAT;
ctx->priority = RT_PRIORITY;
ctx->running = false;
// Allocate buffers
size_t buffer_samples = ctx->period_size * ctx->channels;
ctx->input_buffer = malloc(buffer_samples * sizeof(int32_t));
ctx->output_buffer = malloc(buffer_samples * sizeof(int32_t));
ctx->float_buffer = malloc(buffer_samples * sizeof(float));
if (!ctx->input_buffer || !ctx->output_buffer || !ctx->float_buffer) {
audio_context_destroy(ctx);
return NULL;
}
// Setup ALSA devices
if (setup_alsa_device(ctx, playback_device, SND_PCM_STREAM_PLAYBACK,
&ctx->playback_handle) < 0) {
audio_context_destroy(ctx);
return NULL;
}
if (setup_alsa_device(ctx, capture_device, SND_PCM_STREAM_CAPTURE,
&ctx->capture_handle) < 0) {
audio_context_destroy(ctx);
return NULL;
}
// Set default processing callback
ctx->process_callback = default_process_callback;
printf("Audio context created successfully\n");
return ctx;
}
// Start audio processing
static int audio_context_start(audio_context_t *ctx) {
if (!ctx || ctx->running) return -1;
// Lock memory pages for real-time performance
lock_memory();
// Prepare ALSA devices
if (snd_pcm_prepare(ctx->playback_handle) < 0) {
fprintf(stderr, "Cannot prepare playback interface\n");
return -1;
}
if (snd_pcm_prepare(ctx->capture_handle) < 0) {
fprintf(stderr, "Cannot prepare capture interface\n");
return -1;
}
// Start capture device
if (snd_pcm_start(ctx->capture_handle) < 0) {
fprintf(stderr, "Cannot start capture interface\n");
return -1;
}
ctx->running = true;
// Create audio processing thread
if (pthread_create(&ctx->audio_thread, NULL, audio_thread_func, ctx) != 0) {
fprintf(stderr, "Cannot create audio thread\n");
ctx->running = false;
return -1;
}
printf("Audio processing started\n");
return 0;
}
// Stop audio processing
static void audio_context_stop(audio_context_t *ctx) {
if (!ctx || !ctx->running) return;
ctx->running = false;
// Wait for audio thread to finish
pthread_join(ctx->audio_thread, NULL);
// Stop ALSA devices
snd_pcm_drop(ctx->playback_handle);
snd_pcm_drop(ctx->capture_handle);
printf("Audio processing stopped\n");
}
// Cleanup audio context
static void audio_context_destroy(audio_context_t *ctx) {
if (!ctx) return;
if (ctx->running) {
audio_context_stop(ctx);
}
if (ctx->playback_handle) {
snd_pcm_close(ctx->playback_handle);
}
if (ctx->capture_handle) {
snd_pcm_close(ctx->capture_handle);
}
free(ctx->input_buffer);
free(ctx->output_buffer);
free(ctx->float_buffer);
free(ctx);
}
// Print audio statistics
static void print_audio_stats(audio_context_t *ctx) {
printf("\n=== Audio Statistics ===\n");
printf("Processed frames: %lu\n", ctx->stats.processed_frames);
printf("XRuns: %lu\n", ctx->stats.xruns);
printf("Underruns: %lu\n", ctx->stats.underruns);
printf("Overruns: %lu\n", ctx->stats.overruns);
printf("Average latency: %.2f ms\n", ctx->stats.avg_latency_ms);
printf("Maximum latency: %.2f ms\n", ctx->stats.max_latency_ms);
double runtime_s = (double)ctx->stats.processed_frames / ctx->sample_rate;
printf("Runtime: %.1f seconds\n", runtime_s);
if (runtime_s > 0) {
printf("XRun rate: %.2f/minute\n",
(ctx->stats.underruns + ctx->stats.overruns) / runtime_s * 60.0);
}
}
// Signal handler
static void signal_handler(int sig) {
printf("\nReceived signal %d, shutting down...\n", sig);
g_shutdown = true;
if (g_audio_ctx) {
audio_context_stop(g_audio_ctx);
}
}
// Example DSP processing with effects
static void advanced_process_callback(audio_context_t *ctx, float *input, float *output,
snd_pcm_uframes_t frames) {
static biquad_filter_t lowpass_filter = {0};
static delay_effect_t *delay_effect = NULL;
static bool effects_initialized = false;
if (!effects_initialized) {
biquad_filter_init(&lowpass_filter, 2000.0f, 0.707f, ctx->sample_rate);
delay_effect = delay_effect_create(200.0f, ctx->sample_rate, 0.3f, 0.2f);
effects_initialized = true;
}
for (snd_pcm_uframes_t i = 0; i < frames; i++) {
for (unsigned int ch = 0; ch < ctx->channels; ch++) {
size_t idx = i * ctx->channels + ch;
float sample = input[idx];
// Apply lowpass filter
sample = biquad_filter_process(&lowpass_filter, sample);
// Apply delay effect
if (delay_effect) {
sample = delay_effect_process(delay_effect, sample);
}
// Apply gain and soft limiting
sample *= 0.8f;
if (sample > 0.95f) sample = 0.95f;
if (sample < -0.95f) sample = -0.95f;
output[idx] = sample;
}
}
}
// Main function
int main(int argc, char *argv[]) {
const char *playback_device = "default";
const char *capture_device = "default";
// Parse command line arguments
if (argc > 1) playback_device = argv[1];
if (argc > 2) capture_device = argv[2];
printf("Advanced ALSA Audio Processing\n");
printf("==============================\n");
printf("Playback device: %s\n", playback_device);
printf("Capture device: %s\n", capture_device);
// Install signal handlers
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
// Create audio context
g_audio_ctx = audio_context_create(playback_device, capture_device);
if (!g_audio_ctx) {
fprintf(stderr, "Failed to create audio context\n");
return 1;
}
// Set advanced processing callback
g_audio_ctx->process_callback = advanced_process_callback;
// Start audio processing
if (audio_context_start(g_audio_ctx) < 0) {
fprintf(stderr, "Failed to start audio processing\n");
audio_context_destroy(g_audio_ctx);
return 1;
}
printf("Audio processing running. Press Ctrl+C to stop.\n");
// Print statistics periodically
while (!g_shutdown) {
sleep(5);
if (!g_shutdown) {
print_audio_stats(g_audio_ctx);
}
}
// Cleanup
audio_context_destroy(g_audio_ctx);
printf("Audio processing terminated\n");
return 0;
}
PulseAudio Module Development
Custom PulseAudio Module Implementation
// module_advanced_processor.c - Advanced PulseAudio module
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include <pulsecore/core.h>
#include <pulsecore/module.h>
#include <pulsecore/sink.h>
#include <pulsecore/source.h>
#include <pulsecore/sink-input.h>
#include <pulsecore/source-output.h>
#include <pulsecore/core-util.h>
#include <pulsecore/log.h>
#include <pulsecore/thread.h>
#include <pulsecore/thread-mq.h>
#include <pulsecore/rtpoll.h>
#include <pulsecore/memchunk.h>
#include <pulsecore/resampler.h>
#include <pulse/sample.h>
#include <pulse/util.h>
#include <fftw3.h>
#include <math.h>
#include <string.h>
PA_MODULE_AUTHOR("Matthew Mattox");
PA_MODULE_DESCRIPTION("Advanced Audio Processor Module");
PA_MODULE_VERSION(PACKAGE_VERSION);
PA_MODULE_LOAD_ONCE(false);
PA_MODULE_USAGE(
"sink_name=<name of sink> "
"sink_properties=<properties for the sink> "
"master=<name of sink to filter> "
"rate=<sample rate> "
"channels=<number of channels> "
"channel_map=<channel map> "
"effect=<effect type> "
"effect_params=<effect parameters>");
#define MEMPOOL_SLOT_SIZE (16*1024)
#define DEFAULT_SINK_NAME "advanced_processor"
#define MAX_CHANNELS 8
#define FFT_SIZE 1024
// Effect types
typedef enum {
EFFECT_NONE,
EFFECT_EQUALIZER,
EFFECT_COMPRESSOR,
EFFECT_REVERB,
EFFECT_NOISE_GATE,
EFFECT_SPECTRAL_ENHANCER
} effect_type_t;
// DSP structures
typedef struct {
float gain[10]; // 10-band EQ
float freq[10];
biquad_filter_t filters[MAX_CHANNELS][10];
} equalizer_t;
typedef struct {
float threshold;
float ratio;
float attack_ms;
float release_ms;
float makeup_gain;
float envelope[MAX_CHANNELS];
float attack_coeff;
float release_coeff;
} compressor_t;
typedef struct {
float room_size;
float damping;
float wet_level;
float dry_level;
float *delay_lines[8];
size_t delay_lengths[8];
size_t write_pos[8];
float all_pass_delays[4][MAX_CHANNELS];
size_t all_pass_pos[4];
} reverb_t;
typedef struct {
float threshold;
float ratio;
float attack_ms;
float release_ms;
float hold_ms;
float envelope[MAX_CHANNELS];
size_t hold_samples[MAX_CHANNELS];
float attack_coeff;
float release_coeff;
} noise_gate_t;
typedef struct {
fftw_complex *fft_input[MAX_CHANNELS];
fftw_complex *fft_output[MAX_CHANNELS];
fftw_plan forward_plan[MAX_CHANNELS];
fftw_plan inverse_plan[MAX_CHANNELS];
float *window;
float *overlap_buffer[MAX_CHANNELS];
float enhancement_strength;
} spectral_enhancer_t;
// Module userdata
struct userdata {
pa_core *core;
pa_module *module;
pa_sink *sink;
pa_sink_input *sink_input;
pa_memblockq *memblockq;
bool auto_desc;
// Processing parameters
effect_type_t effect_type;
uint32_t sample_rate;
uint8_t channels;
pa_sample_spec sample_spec;
pa_channel_map channel_map;
// DSP processing
union {
equalizer_t equalizer;
compressor_t compressor;
reverb_t reverb;
noise_gate_t noise_gate;
spectral_enhancer_t spectral_enhancer;
} effect;
// Performance monitoring
uint64_t processed_samples;
pa_usec_t processing_time;
};
static const char* const valid_modargs[] = {
"sink_name",
"sink_properties",
"master",
"rate",
"channels",
"channel_map",
"effect",
"effect_params",
NULL
};
// DSP processing functions
static void equalizer_init(equalizer_t *eq, uint32_t sample_rate) {
// Initialize 10-band equalizer with standard frequencies
float frequencies[] = {31.5f, 63.0f, 125.0f, 250.0f, 500.0f,
1000.0f, 2000.0f, 4000.0f, 8000.0f, 16000.0f};
for (int band = 0; band < 10; band++) {
eq->freq[band] = frequencies[band];
eq->gain[band] = 1.0f; // Unity gain initially
for (int ch = 0; ch < MAX_CHANNELS; ch++) {
biquad_filter_init(&eq->filters[ch][band], frequencies[band],
0.707f, sample_rate);
}
}
}
static void equalizer_process(equalizer_t *eq, float *samples, size_t frames,
uint8_t channels) {
for (size_t frame = 0; frame < frames; frame++) {
for (uint8_t ch = 0; ch < channels; ch++) {
float sample = samples[frame * channels + ch];
// Apply all EQ bands
for (int band = 0; band < 10; band++) {
sample = biquad_filter_process(&eq->filters[ch][band], sample);
sample *= eq->gain[band];
}
samples[frame * channels + ch] = sample;
}
}
}
static void compressor_init(compressor_t *comp, uint32_t sample_rate) {
comp->threshold = -20.0f; // dB
comp->ratio = 4.0f;
comp->attack_ms = 5.0f;
comp->release_ms = 100.0f;
comp->makeup_gain = 1.0f;
// Calculate filter coefficients
comp->attack_coeff = expf(-1.0f / (comp->attack_ms * sample_rate / 1000.0f));
comp->release_coeff = expf(-1.0f / (comp->release_ms * sample_rate / 1000.0f));
for (int ch = 0; ch < MAX_CHANNELS; ch++) {
comp->envelope[ch] = 0.0f;
}
}
static void compressor_process(compressor_t *comp, float *samples, size_t frames,
uint8_t channels) {
float threshold_linear = powf(10.0f, comp->threshold / 20.0f);
for (size_t frame = 0; frame < frames; frame++) {
for (uint8_t ch = 0; ch < channels; ch++) {
float sample = samples[frame * channels + ch];
float abs_sample = fabsf(sample);
// Envelope follower
float target = abs_sample > comp->envelope[ch] ? abs_sample : comp->envelope[ch];
float coeff = abs_sample > comp->envelope[ch] ? comp->attack_coeff : comp->release_coeff;
comp->envelope[ch] = target + (comp->envelope[ch] - target) * coeff;
// Compression
if (comp->envelope[ch] > threshold_linear) {
float excess = comp->envelope[ch] / threshold_linear;
float compressed_excess = powf(excess, 1.0f / comp->ratio);
float gain_reduction = compressed_excess / excess;
sample *= gain_reduction;
}
// Makeup gain
sample *= comp->makeup_gain;
samples[frame * channels + ch] = sample;
}
}
}
static void reverb_init(reverb_t *rev, uint32_t sample_rate) {
rev->room_size = 0.5f;
rev->damping = 0.5f;
rev->wet_level = 0.3f;
rev->dry_level = 0.7f;
// Initialize delay lines for early reflections
size_t delay_times[] = {347, 113, 37, 59, 53, 43, 37, 29}; // Prime numbers
for (int i = 0; i < 8; i++) {
rev->delay_lengths[i] = (delay_times[i] * sample_rate) / 1000;
rev->delay_lines[i] = pa_xmalloc0(rev->delay_lengths[i] * sizeof(float));
rev->write_pos[i] = 0;
}
// Initialize allpass delays
for (int i = 0; i < 4; i++) {
for (int ch = 0; ch < MAX_CHANNELS; ch++) {
rev->all_pass_delays[i][ch] = 0.0f;
}
rev->all_pass_pos[i] = 0;
}
}
static void reverb_process(reverb_t *rev, float *samples, size_t frames, uint8_t channels) {
for (size_t frame = 0; frame < frames; frame++) {
for (uint8_t ch = 0; ch < channels; ch++) {
float input = samples[frame * channels + ch];
float output = 0.0f;
// Early reflections
for (int i = 0; i < 8; i++) {
size_t read_pos = (rev->write_pos[i] + rev->delay_lengths[i] -
(size_t)(rev->delay_lengths[i] * rev->room_size)) %
rev->delay_lengths[i];
float delayed = rev->delay_lines[i][read_pos];
output += delayed * 0.125f; // Mix 8 delays
// Feedback with damping
rev->delay_lines[i][rev->write_pos[i]] = input + delayed *
(1.0f - rev->damping) * 0.5f;
rev->write_pos[i] = (rev->write_pos[i] + 1) % rev->delay_lengths[i];
}
// Mix wet and dry signals
float final_output = input * rev->dry_level + output * rev->wet_level;
samples[frame * channels + ch] = final_output;
}
}
}
static void reverb_cleanup(reverb_t *rev) {
for (int i = 0; i < 8; i++) {
if (rev->delay_lines[i]) {
pa_xfree(rev->delay_lines[i]);
}
}
}
// Main audio processing function
static void process_audio(struct userdata *u, const pa_memchunk *chunk) {
void *src, *dst;
size_t n_frames;
pa_memchunk tchunk;
pa_assert(u);
pa_assert(chunk);
// Get audio data
src = pa_memblock_acquire(chunk->memblock);
n_frames = chunk->length / pa_frame_size(&u->sample_spec);
// Create output chunk
tchunk.memblock = pa_memblock_new(u->core->mempool, chunk->length);
tchunk.index = 0;
tchunk.length = chunk->length;
dst = pa_memblock_acquire(tchunk.memblock);
// Copy input to output for processing
memcpy(dst, (uint8_t*)src + chunk->index, chunk->length);
// Apply DSP processing based on effect type
float *samples = (float*)dst;
switch (u->effect_type) {
case EFFECT_EQUALIZER:
equalizer_process(&u->effect.equalizer, samples, n_frames, u->channels);
break;
case EFFECT_COMPRESSOR:
compressor_process(&u->effect.compressor, samples, n_frames, u->channels);
break;
case EFFECT_REVERB:
reverb_process(&u->effect.reverb, samples, n_frames, u->channels);
break;
case EFFECT_NOISE_GATE:
// Implementation similar to compressor but with gating
break;
case EFFECT_SPECTRAL_ENHANCER:
// FFT-based spectral enhancement
break;
case EFFECT_NONE:
default:
// Pass through
break;
}
pa_memblock_release(chunk->memblock);
pa_memblock_release(tchunk.memblock);
// Update statistics
u->processed_samples += n_frames;
// Push processed audio to sink
pa_sink_render_into(u->sink, &tchunk);
pa_memblock_unref(tchunk.memblock);
}
// Sink input callbacks
static int sink_input_pop_cb(pa_sink_input *i, size_t nbytes, pa_memchunk *chunk) {
struct userdata *u;
pa_sink_input_assert_ref(i);
pa_assert_se(u = i->userdata);
pa_assert(chunk);
// Get audio from master sink
if (pa_sink_render(u->sink_input->sink, nbytes, chunk) < 0)
return -1;
// Process the audio
process_audio(u, chunk);
return 0;
}
static void sink_input_process_rewind_cb(pa_sink_input *i, size_t nbytes) {
struct userdata *u;
pa_sink_input_assert_ref(i);
pa_assert_se(u = i->userdata);
pa_sink_process_rewind(u->sink, nbytes);
}
static void sink_input_update_max_rewind_cb(pa_sink_input *i, size_t nbytes) {
struct userdata *u;
pa_sink_input_assert_ref(i);
pa_assert_se(u = i->userdata);
pa_sink_set_max_rewind_within_thread(u->sink, nbytes);
}
static void sink_input_update_max_request_cb(pa_sink_input *i, size_t nbytes) {
struct userdata *u;
pa_sink_input_assert_ref(i);
pa_assert_se(u = i->userdata);
pa_sink_set_max_request_within_thread(u->sink, nbytes);
}
static void sink_input_update_sink_latency_range_cb(pa_sink_input *i) {
struct userdata *u;
pa_sink_input_assert_ref(i);
pa_assert_se(u = i->userdata);
pa_sink_set_latency_range_within_thread(u->sink,
i->sink->thread_info.min_latency,
i->sink->thread_info.max_latency);
}
static void sink_input_update_sink_fixed_latency_cb(pa_sink_input *i) {
struct userdata *u;
pa_sink_input_assert_ref(i);
pa_assert_se(u = i->userdata);
pa_sink_set_fixed_latency_within_thread(u->sink, i->sink->thread_info.fixed_latency);
}
static void sink_input_detach_cb(pa_sink_input *i) {
struct userdata *u;
pa_sink_input_assert_ref(i);
pa_assert_se(u = i->userdata);
pa_sink_detach_within_thread(u->sink);
pa_sink_set_rtpoll(u->sink, NULL);
}
static void sink_input_attach_cb(pa_sink_input *i) {
struct userdata *u;
pa_sink_input_assert_ref(i);
pa_assert_se(u = i->userdata);
pa_sink_set_rtpoll(u->sink, i->sink->thread_info.rtpoll);
pa_sink_attach_within_thread(u->sink);
}
static void sink_input_kill_cb(pa_sink_input *i) {
struct userdata *u;
pa_sink_input_assert_ref(i);
pa_assert_se(u = i->userdata);
pa_sink_unlink(u->sink);
pa_sink_input_unlink(u->sink_input);
pa_sink_input_unref(u->sink_input);
u->sink_input = NULL;
pa_sink_unref(u->sink);
u->sink = NULL;
pa_module_unload_request(u->module, true);
}
// Module load function
int pa__init(pa_module*m) {
struct userdata *u;
pa_sample_spec ss;
pa_channel_map map;
pa_modargs *ma;
pa_sink *master;
pa_sink_input_new_data sink_input_data;
pa_sink_new_data sink_data;
const char *effect_str;
pa_assert(m);
if (!(ma = pa_modargs_new(m->argument, valid_modargs))) {
pa_log("Failed to parse module arguments.");
goto fail;
}
if (!(master = pa_namereg_get(m->core, pa_modargs_get_value(ma, "master", NULL),
PA_NAMEREG_SINK))) {
pa_log("Master sink not found");
goto fail;
}
ss = master->sample_spec;
map = master->channel_map;
if (pa_modargs_get_sample_spec_and_channel_map(ma, &ss, &map,
PA_CHANNEL_MAP_DEFAULT) < 0) {
pa_log("Invalid sample format specification or channel map");
goto fail;
}
u = pa_xnew0(struct userdata, 1);
u->core = m->core;
u->module = m;
u->sample_spec = ss;
u->channel_map = map;
u->channels = ss.channels;
u->sample_rate = ss.rate;
// Parse effect type
effect_str = pa_modargs_get_value(ma, "effect", "none");
if (pa_streq(effect_str, "equalizer")) {
u->effect_type = EFFECT_EQUALIZER;
equalizer_init(&u->effect.equalizer, u->sample_rate);
} else if (pa_streq(effect_str, "compressor")) {
u->effect_type = EFFECT_COMPRESSOR;
compressor_init(&u->effect.compressor, u->sample_rate);
} else if (pa_streq(effect_str, "reverb")) {
u->effect_type = EFFECT_REVERB;
reverb_init(&u->effect.reverb, u->sample_rate);
} else {
u->effect_type = EFFECT_NONE;
}
m->userdata = u;
// Create sink
pa_sink_new_data_init(&sink_data);
sink_data.driver = __FILE__;
sink_data.module = m;
if (!(sink_data.name = pa_xstrdup(pa_modargs_get_value(ma, "sink_name",
DEFAULT_SINK_NAME)))) {
pa_log("sink_name= expects a sink name");
goto fail;
}
pa_sink_new_data_set_sample_spec(&sink_data, &ss);
pa_sink_new_data_set_channel_map(&sink_data, &map);
pa_proplist_sets(sink_data.proplist, PA_PROP_DEVICE_MASTER_DEVICE, master->name);
pa_proplist_sets(sink_data.proplist, PA_PROP_DEVICE_CLASS, "filter");
pa_proplist_sets(sink_data.proplist, PA_PROP_DEVICE_DESCRIPTION,
"Advanced Audio Processor");
if (pa_modargs_get_proplist(ma, "sink_properties", sink_data.proplist,
PA_UPDATE_REPLACE) < 0) {
pa_log("Invalid properties");
pa_sink_new_data_done(&sink_data);
goto fail;
}
u->sink = pa_sink_new(m->core, &sink_data,
PA_SINK_LATENCY | PA_SINK_DYNAMIC_LATENCY);
pa_sink_new_data_done(&sink_data);
if (!u->sink) {
pa_log("Failed to create sink.");
goto fail;
}
u->sink->parent.process_msg = NULL; // We don't need this
u->sink->userdata = u;
pa_sink_set_asyncmsgq(u->sink, master->asyncmsgq);
pa_sink_set_rtpoll(u->sink, master->rtpoll);
// Create sink input
pa_sink_input_new_data_init(&sink_input_data);
sink_input_data.driver = __FILE__;
sink_input_data.module = m;
sink_input_data.sink = master;
pa_sink_input_new_data_set_sample_spec(&sink_input_data, &ss);
pa_sink_input_new_data_set_channel_map(&sink_input_data, &map);
pa_proplist_sets(sink_input_data.proplist, PA_PROP_MEDIA_NAME,
"Advanced Audio Processor Stream");
pa_proplist_sets(sink_input_data.proplist, PA_PROP_MEDIA_ROLE, "filter");
pa_sink_input_new(&u->sink_input, m->core, &sink_input_data);
pa_sink_input_new_data_done(&sink_input_data);
if (!u->sink_input) {
pa_log("Failed to create sink input.");
goto fail;
}
u->sink_input->pop = sink_input_pop_cb;
u->sink_input->process_rewind = sink_input_process_rewind_cb;
u->sink_input->update_max_rewind = sink_input_update_max_rewind_cb;
u->sink_input->update_max_request = sink_input_update_max_request_cb;
u->sink_input->update_sink_latency_range = sink_input_update_sink_latency_range_cb;
u->sink_input->update_sink_fixed_latency = sink_input_update_sink_fixed_latency_cb;
u->sink_input->kill = sink_input_kill_cb;
u->sink_input->attach = sink_input_attach_cb;
u->sink_input->detach = sink_input_detach_cb;
u->sink_input->userdata = u;
pa_sink_put(u->sink);
pa_sink_input_put(u->sink_input);
pa_modargs_free(ma);
pa_log_info("Advanced audio processor module loaded successfully");
return 0;
fail:
if (ma)
pa_modargs_free(ma);
pa__done(m);
return -1;
}
// Module unload function
void pa__done(pa_module*m) {
struct userdata *u;
pa_assert(m);
if (!(u = m->userdata))
return;
if (u->sink_input) {
pa_sink_input_unlink(u->sink_input);
pa_sink_input_unref(u->sink_input);
}
if (u->sink) {
pa_sink_unlink(u->sink);
pa_sink_unref(u->sink);
}
// Cleanup effects
if (u->effect_type == EFFECT_REVERB) {
reverb_cleanup(&u->effect.reverb);
}
pa_log_info("Advanced audio processor module unloaded (processed %lu samples)",
u->processed_samples);
pa_xfree(u);
}
// Module get author function
int pa__get_author(pa_module *m, const char **author) {
pa_assert(m);
pa_assert(author);
*author = PA_MODULE_AUTHOR;
return 0;
}
// Module get description function
int pa__get_description(pa_module *m, const char **description) {
pa_assert(m);
pa_assert(description);
*description = PA_MODULE_DESCRIPTION;
return 0;
}
// Module get usage function
int pa__get_usage(pa_module *m, const char **usage) {
pa_assert(m);
pa_assert(usage);
*usage = PA_MODULE_USAGE;
return 0;
}
// Module get version function
int pa__get_version(pa_module *m, const char **version) {
pa_assert(m);
pa_assert(version);
*version = PA_MODULE_VERSION;
return 0;
}
FFmpeg Integration and Media Processing
Advanced FFmpeg Media Processing Framework
// ffmpeg_advanced.c - Advanced FFmpeg media processing framework
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <pthread.h>
#include <time.h>
#include <math.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/opt.h>
#include <libavutil/imgutils.h>
#include <libavutil/samplefmt.h>
#include <libavutil/timestamp.h>
#include <libavfilter/avfilter.h>
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
#include <libswscale/swscale.h>
#include <libswresample/swresample.h>
#include <libavdevice/avdevice.h>
#define MAX_STREAMS 16
#define MAX_FILTERS 32
#define BUFFER_SIZE 4096
// Media processing context
typedef struct {
AVFormatContext *input_ctx;
AVFormatContext *output_ctx;
// Stream information
int video_stream_idx;
int audio_stream_idx;
AVCodecContext *video_dec_ctx;
AVCodecContext *audio_dec_ctx;
AVCodecContext *video_enc_ctx;
AVCodecContext *audio_enc_ctx;
// Filter graph
AVFilterGraph *filter_graph;
AVFilterContext *video_src_ctx;
AVFilterContext *video_sink_ctx;
AVFilterContext *audio_src_ctx;
AVFilterContext *audio_sink_ctx;
// Processing parameters
char input_filename[256];
char output_filename[256];
char video_filter_desc[1024];
char audio_filter_desc[1024];
// Performance monitoring
int64_t processed_frames;
int64_t total_frames;
double processing_fps;
time_t start_time;
// Threading
pthread_t processing_thread;
bool running;
} media_context_t;
// Global context
static media_context_t *g_media_ctx = NULL;
// Error handling
static void log_error(const char *fmt, ...) {
va_list args;
va_start(args, fmt);
fprintf(stderr, "ERROR: ");
vfprintf(stderr, fmt, args);
fprintf(stderr, "\n");
va_end(args);
}
static char *av_error_string(int errnum) {
static char str[AV_ERROR_MAX_STRING_SIZE];
return av_make_error_string(str, AV_ERROR_MAX_STRING_SIZE, errnum);
}
// Initialize FFmpeg libraries
static int init_ffmpeg(void) {
av_log_set_level(AV_LOG_INFO);
// Register all codecs and formats
avcodec_register_all();
av_register_all();
avfilter_register_all();
avdevice_register_all();
printf("FFmpeg initialized successfully\n");
printf("Codecs: %d, Formats: %d, Filters: %d\n",
avcodec_get_count(), av_format_get_count(), avfilter_get_count());
return 0;
}
// Open input file and find streams
static int open_input_file(media_context_t *ctx) {
int ret;
// Open input file
ret = avformat_open_input(&ctx->input_ctx, ctx->input_filename, NULL, NULL);
if (ret < 0) {
log_error("Cannot open input file '%s': %s",
ctx->input_filename, av_error_string(ret));
return ret;
}
// Retrieve stream information
ret = avformat_find_stream_info(ctx->input_ctx, NULL);
if (ret < 0) {
log_error("Cannot find stream information: %s", av_error_string(ret));
return ret;
}
// Find video and audio streams
ctx->video_stream_idx = av_find_best_stream(ctx->input_ctx, AVMEDIA_TYPE_VIDEO,
-1, -1, NULL, 0);
ctx->audio_stream_idx = av_find_best_stream(ctx->input_ctx, AVMEDIA_TYPE_AUDIO,
-1, -1, NULL, 0);
if (ctx->video_stream_idx >= 0) {
AVStream *video_stream = ctx->input_ctx->streams[ctx->video_stream_idx];
AVCodec *video_decoder = avcodec_find_decoder(video_stream->codecpar->codec_id);
if (!video_decoder) {
log_error("Failed to find video decoder");
return AVERROR(EINVAL);
}
ctx->video_dec_ctx = avcodec_alloc_context3(video_decoder);
if (!ctx->video_dec_ctx) {
log_error("Failed to allocate video decoder context");
return AVERROR(ENOMEM);
}
ret = avcodec_parameters_to_context(ctx->video_dec_ctx, video_stream->codecpar);
if (ret < 0) {
log_error("Failed to copy video decoder parameters: %s", av_error_string(ret));
return ret;
}
ret = avcodec_open2(ctx->video_dec_ctx, video_decoder, NULL);
if (ret < 0) {
log_error("Failed to open video decoder: %s", av_error_string(ret));
return ret;
}
printf("Video stream found: %dx%d, %s, %.2f fps\n",
ctx->video_dec_ctx->width, ctx->video_dec_ctx->height,
av_get_pix_fmt_name(ctx->video_dec_ctx->pix_fmt),
av_q2d(video_stream->r_frame_rate));
}
if (ctx->audio_stream_idx >= 0) {
AVStream *audio_stream = ctx->input_ctx->streams[ctx->audio_stream_idx];
AVCodec *audio_decoder = avcodec_find_decoder(audio_stream->codecpar->codec_id);
if (!audio_decoder) {
log_error("Failed to find audio decoder");
return AVERROR(EINVAL);
}
ctx->audio_dec_ctx = avcodec_alloc_context3(audio_decoder);
if (!ctx->audio_dec_ctx) {
log_error("Failed to allocate audio decoder context");
return AVERROR(ENOMEM);
}
ret = avcodec_parameters_to_context(ctx->audio_dec_ctx, audio_stream->codecpar);
if (ret < 0) {
log_error("Failed to copy audio decoder parameters: %s", av_error_string(ret));
return ret;
}
ret = avcodec_open2(ctx->audio_dec_ctx, audio_decoder, NULL);
if (ret < 0) {
log_error("Failed to open audio decoder: %s", av_error_string(ret));
return ret;
}
printf("Audio stream found: %d Hz, %d channels, %s\n",
ctx->audio_dec_ctx->sample_rate, ctx->audio_dec_ctx->channels,
av_get_sample_fmt_name(ctx->audio_dec_ctx->sample_fmt));
}
// Print input file information
av_dump_format(ctx->input_ctx, 0, ctx->input_filename, 0);
return 0;
}
// Create output file and encoders
static int create_output_file(media_context_t *ctx) {
int ret;
AVStream *out_stream;
AVCodec *encoder;
// Allocate output format context
avformat_alloc_output_context2(&ctx->output_ctx, NULL, NULL, ctx->output_filename);
if (!ctx->output_ctx) {
log_error("Could not create output context");
return AVERROR_UNKNOWN;
}
// Create video stream and encoder
if (ctx->video_stream_idx >= 0) {
encoder = avcodec_find_encoder(AV_CODEC_ID_H264);
if (!encoder) {
log_error("H264 encoder not found");
return AVERROR_INVALIDDATA;
}
out_stream = avformat_new_stream(ctx->output_ctx, NULL);
if (!out_stream) {
log_error("Failed allocating output video stream");
return AVERROR_UNKNOWN;
}
ctx->video_enc_ctx = avcodec_alloc_context3(encoder);
if (!ctx->video_enc_ctx) {
log_error("Failed to allocate video encoder context");
return AVERROR(ENOMEM);
}
// Set video encoder parameters
ctx->video_enc_ctx->height = ctx->video_dec_ctx->height;
ctx->video_enc_ctx->width = ctx->video_dec_ctx->width;
ctx->video_enc_ctx->sample_aspect_ratio = ctx->video_dec_ctx->sample_aspect_ratio;
ctx->video_enc_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
ctx->video_enc_ctx->time_base = av_inv_q(ctx->input_ctx->streams[ctx->video_stream_idx]->r_frame_rate);
// Codec-specific settings
ctx->video_enc_ctx->bit_rate = 2000000; // 2 Mbps
ctx->video_enc_ctx->rc_buffer_size = 4000000;
ctx->video_enc_ctx->rc_max_rate = 2000000;
ctx->video_enc_ctx->rc_min_rate = 500000;
ctx->video_enc_ctx->gop_size = 50;
ctx->video_enc_ctx->max_b_frames = 2;
// Quality settings
av_opt_set(ctx->video_enc_ctx->priv_data, "preset", "medium", 0);
av_opt_set(ctx->video_enc_ctx->priv_data, "crf", "23", 0);
if (ctx->output_ctx->oformat->flags & AVFMT_GLOBALHEADER) {
ctx->video_enc_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
ret = avcodec_open2(ctx->video_enc_ctx, encoder, NULL);
if (ret < 0) {
log_error("Cannot open video encoder: %s", av_error_string(ret));
return ret;
}
ret = avcodec_parameters_from_context(out_stream->codecpar, ctx->video_enc_ctx);
if (ret < 0) {
log_error("Failed to copy video encoder parameters: %s", av_error_string(ret));
return ret;
}
out_stream->time_base = ctx->video_enc_ctx->time_base;
}
// Create audio stream and encoder
if (ctx->audio_stream_idx >= 0) {
encoder = avcodec_find_encoder(AV_CODEC_ID_AAC);
if (!encoder) {
log_error("AAC encoder not found");
return AVERROR_INVALIDDATA;
}
out_stream = avformat_new_stream(ctx->output_ctx, NULL);
if (!out_stream) {
log_error("Failed allocating output audio stream");
return AVERROR_UNKNOWN;
}
ctx->audio_enc_ctx = avcodec_alloc_context3(encoder);
if (!ctx->audio_enc_ctx) {
log_error("Failed to allocate audio encoder context");
return AVERROR(ENOMEM);
}
// Set audio encoder parameters
ctx->audio_enc_ctx->channels = ctx->audio_dec_ctx->channels;
ctx->audio_enc_ctx->channel_layout = av_get_default_channel_layout(ctx->audio_dec_ctx->channels);
ctx->audio_enc_ctx->sample_rate = ctx->audio_dec_ctx->sample_rate;
ctx->audio_enc_ctx->sample_fmt = encoder->sample_fmts[0];
ctx->audio_enc_ctx->bit_rate = 128000; // 128 kbps
ctx->audio_enc_ctx->time_base = (AVRational){1, ctx->audio_enc_ctx->sample_rate};
if (ctx->output_ctx->oformat->flags & AVFMT_GLOBALHEADER) {
ctx->audio_enc_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
ret = avcodec_open2(ctx->audio_enc_ctx, encoder, NULL);
if (ret < 0) {
log_error("Cannot open audio encoder: %s", av_error_string(ret));
return ret;
}
ret = avcodec_parameters_from_context(out_stream->codecpar, ctx->audio_enc_ctx);
if (ret < 0) {
log_error("Failed to copy audio encoder parameters: %s", av_error_string(ret));
return ret;
}
out_stream->time_base = ctx->audio_enc_ctx->time_base;
}
// Print output file information
av_dump_format(ctx->output_ctx, 0, ctx->output_filename, 1);
// Open output file
if (!(ctx->output_ctx->oformat->flags & AVFMT_NOFILE)) {
ret = avio_open(&ctx->output_ctx->pb, ctx->output_filename, AVIO_FLAG_WRITE);
if (ret < 0) {
log_error("Could not open output file '%s': %s",
ctx->output_filename, av_error_string(ret));
return ret;
}
}
// Write file header
ret = avformat_write_header(ctx->output_ctx, NULL);
if (ret < 0) {
log_error("Error occurred when opening output file: %s", av_error_string(ret));
return ret;
}
return 0;
}
// Initialize filter graph
static int init_filter_graph(media_context_t *ctx) {
char args[512];
int ret;
const AVFilter *buffersrc, *buffersink;
AVFilterInOut *outputs, *inputs;
// Create filter graph
ctx->filter_graph = avfilter_graph_alloc();
if (!ctx->filter_graph) {
log_error("Cannot allocate filter graph");
return AVERROR(ENOMEM);
}
// Video filter setup
if (ctx->video_stream_idx >= 0 && strlen(ctx->video_filter_desc) > 0) {
buffersrc = avfilter_get_by_name("buffer");
buffersink = avfilter_get_by_name("buffersink");
outputs = avfilter_inout_alloc();
inputs = avfilter_inout_alloc();
if (!outputs || !inputs || !buffersrc || !buffersink) {
ret = AVERROR(ENOMEM);
goto end;
}
// Create buffer source
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
ctx->video_dec_ctx->width, ctx->video_dec_ctx->height,
ctx->video_dec_ctx->pix_fmt,
ctx->video_dec_ctx->time_base.num, ctx->video_dec_ctx->time_base.den,
ctx->video_dec_ctx->sample_aspect_ratio.num,
ctx->video_dec_ctx->sample_aspect_ratio.den);
ret = avfilter_graph_create_filter(&ctx->video_src_ctx, buffersrc, "in",
args, NULL, ctx->filter_graph);
if (ret < 0) {
log_error("Cannot create video buffer source: %s", av_error_string(ret));
goto end;
}
// Create buffer sink
ret = avfilter_graph_create_filter(&ctx->video_sink_ctx, buffersink, "out",
NULL, NULL, ctx->filter_graph);
if (ret < 0) {
log_error("Cannot create video buffer sink: %s", av_error_string(ret));
goto end;
}
// Set output pixel format
enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE };
ret = av_opt_set_int_list(ctx->video_sink_ctx, "pix_fmts", pix_fmts,
AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
log_error("Cannot set output pixel format: %s", av_error_string(ret));
goto end;
}
// Set endpoints for the filter graph
outputs->name = av_strdup("in");
outputs->filter_ctx = ctx->video_src_ctx;
outputs->pad_idx = 0;
outputs->next = NULL;
inputs->name = av_strdup("out");
inputs->filter_ctx = ctx->video_sink_ctx;
inputs->pad_idx = 0;
inputs->next = NULL;
// Parse filter description
ret = avfilter_graph_parse_ptr(ctx->filter_graph, ctx->video_filter_desc,
&inputs, &outputs, NULL);
if (ret < 0) {
log_error("Cannot parse video filter graph: %s", av_error_string(ret));
goto end;
}
// Configure filter graph
ret = avfilter_graph_config(ctx->filter_graph, NULL);
if (ret < 0) {
log_error("Cannot configure video filter graph: %s", av_error_string(ret));
goto end;
}
printf("Video filter graph initialized: %s\n", ctx->video_filter_desc);
end:
avfilter_inout_free(&inputs);
avfilter_inout_free(&outputs);
if (ret < 0)
return ret;
}
return 0;
}
// Process video frame through filter
static int filter_encode_write_video_frame(media_context_t *ctx, AVFrame *frame) {
int ret;
AVFrame *filt_frame;
// Push frame to filter graph
ret = av_buffersrc_add_frame_flags(ctx->video_src_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF);
if (ret < 0) {
log_error("Error submitting frame to video filter: %s", av_error_string(ret));
return ret;
}
// Pull filtered frames from filter graph
while (1) {
filt_frame = av_frame_alloc();
if (!filt_frame) {
ret = AVERROR(ENOMEM);
break;
}
ret = av_buffersink_get_frame(ctx->video_sink_ctx, filt_frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
av_frame_free(&filt_frame);
break;
}
if (ret < 0) {
av_frame_free(&filt_frame);
log_error("Error getting frame from video filter: %s", av_error_string(ret));
break;
}
filt_frame->pict_type = AV_PICTURE_TYPE_NONE;
// Encode filtered frame
ret = encode_write_frame(ctx, filt_frame, ctx->video_enc_ctx, 0);
av_frame_free(&filt_frame);
if (ret < 0)
break;
}
return ret;
}
// Encode and write frame
static int encode_write_frame(media_context_t *ctx, AVFrame *frame,
AVCodecContext *enc_ctx, int stream_index) {
int ret;
AVPacket enc_pkt;
av_init_packet(&enc_pkt);
enc_pkt.data = NULL;
enc_pkt.size = 0;
// Send frame to encoder
ret = avcodec_send_frame(enc_ctx, frame);
if (ret < 0) {
log_error("Error sending frame to encoder: %s", av_error_string(ret));
return ret;
}
// Receive encoded packets
while (ret >= 0) {
ret = avcodec_receive_packet(enc_ctx, &enc_pkt);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
} else if (ret < 0) {
log_error("Error receiving packet from encoder: %s", av_error_string(ret));
return ret;
}
// Rescale timestamp
av_packet_rescale_ts(&enc_pkt, enc_ctx->time_base,
ctx->output_ctx->streams[stream_index]->time_base);
enc_pkt.stream_index = stream_index;
// Write packet to output
ret = av_interleaved_write_frame(ctx->output_ctx, &enc_pkt);
av_packet_unref(&enc_pkt);
if (ret < 0) {
log_error("Error writing packet: %s", av_error_string(ret));
return ret;
}
ctx->processed_frames++;
}
return 0;
}
// Main processing loop
static void* processing_thread_func(void *arg) {
media_context_t *ctx = (media_context_t*)arg;
AVPacket packet = { .data = NULL, .size = 0 };
AVFrame *frame, *decoded_frame;
int ret;
frame = av_frame_alloc();
decoded_frame = av_frame_alloc();
if (!frame || !decoded_frame) {
log_error("Could not allocate frame");
return NULL;
}
printf("Starting media processing...\n");
// Main processing loop
while (av_read_frame(ctx->input_ctx, &packet) >= 0 && ctx->running) {
if (packet.stream_index == ctx->video_stream_idx) {
// Decode video frame
ret = avcodec_send_packet(ctx->video_dec_ctx, &packet);
if (ret < 0) {
log_error("Error sending video packet: %s", av_error_string(ret));
break;
}
while (ret >= 0) {
ret = avcodec_receive_frame(ctx->video_dec_ctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
} else if (ret < 0) {
log_error("Error receiving video frame: %s", av_error_string(ret));
goto end;
}
// Process frame through filter if enabled
if (ctx->filter_graph) {
ret = filter_encode_write_video_frame(ctx, frame);
} else {
ret = encode_write_frame(ctx, frame, ctx->video_enc_ctx, 0);
}
if (ret < 0)
goto end;
}
} else if (packet.stream_index == ctx->audio_stream_idx) {
// Decode audio frame
ret = avcodec_send_packet(ctx->audio_dec_ctx, &packet);
if (ret < 0) {
log_error("Error sending audio packet: %s", av_error_string(ret));
break;
}
while (ret >= 0) {
ret = avcodec_receive_frame(ctx->audio_dec_ctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
} else if (ret < 0) {
log_error("Error receiving audio frame: %s", av_error_string(ret));
goto end;
}
ret = encode_write_frame(ctx, frame, ctx->audio_enc_ctx, 1);
if (ret < 0)
goto end;
}
}
av_packet_unref(&packet);
// Update progress
if (ctx->processed_frames % 100 == 0) {
time_t current_time = time(NULL);
double elapsed = difftime(current_time, ctx->start_time);
if (elapsed > 0) {
ctx->processing_fps = ctx->processed_frames / elapsed;
printf("Processed %ld frames, %.1f fps\r",
ctx->processed_frames, ctx->processing_fps);
fflush(stdout);
}
}
}
// Flush encoders
if (ctx->video_enc_ctx) {
encode_write_frame(ctx, NULL, ctx->video_enc_ctx, 0);
}
if (ctx->audio_enc_ctx) {
encode_write_frame(ctx, NULL, ctx->audio_enc_ctx, 1);
}
// Write trailer
av_write_trailer(ctx->output_ctx);
end:
av_frame_free(&frame);
av_frame_free(&decoded_frame);
av_packet_unref(&packet);
printf("\nProcessing completed: %ld frames processed\n", ctx->processed_frames);
return NULL;
}
// Initialize media context
static media_context_t* media_context_create(void) {
media_context_t *ctx = calloc(1, sizeof(media_context_t));
if (!ctx) return NULL;
ctx->video_stream_idx = -1;
ctx->audio_stream_idx = -1;
ctx->running = false;
ctx->start_time = time(NULL);
// Set default filter descriptions
strcpy(ctx->video_filter_desc, "scale=1280:720,hqdn3d=4:3:6:4.5");
strcpy(ctx->audio_filter_desc, "");
return ctx;
}
// Cleanup media context
static void media_context_destroy(media_context_t *ctx) {
if (!ctx) return;
if (ctx->running) {
ctx->running = false;
pthread_join(ctx->processing_thread, NULL);
}
if (ctx->filter_graph) {
avfilter_graph_free(&ctx->filter_graph);
}
if (ctx->video_dec_ctx) {
avcodec_free_context(&ctx->video_dec_ctx);
}
if (ctx->audio_dec_ctx) {
avcodec_free_context(&ctx->audio_dec_ctx);
}
if (ctx->video_enc_ctx) {
avcodec_free_context(&ctx->video_enc_ctx);
}
if (ctx->audio_enc_ctx) {
avcodec_free_context(&ctx->audio_enc_ctx);
}
if (ctx->input_ctx) {
avformat_close_input(&ctx->input_ctx);
}
if (ctx->output_ctx) {
if (!(ctx->output_ctx->oformat->flags & AVFMT_NOFILE)) {
avio_closep(&ctx->output_ctx->pb);
}
avformat_free_context(ctx->output_ctx);
}
free(ctx);
}
// Main function
int main(int argc, char *argv[]) {
if (argc < 3) {
printf("Usage: %s <input> <output> [video_filter]\n", argv[0]);
printf("Example: %s input.mp4 output.mp4 \"scale=1280:720,hqdn3d\"\n", argv[0]);
return 1;
}
// Initialize FFmpeg
if (init_ffmpeg() < 0) {
return 1;
}
// Create media context
g_media_ctx = media_context_create();
if (!g_media_ctx) {
log_error("Failed to create media context");
return 1;
}
// Set input/output files
strncpy(g_media_ctx->input_filename, argv[1], sizeof(g_media_ctx->input_filename) - 1);
strncpy(g_media_ctx->output_filename, argv[2], sizeof(g_media_ctx->output_filename) - 1);
if (argc > 3) {
strncpy(g_media_ctx->video_filter_desc, argv[3],
sizeof(g_media_ctx->video_filter_desc) - 1);
}
printf("Input: %s\n", g_media_ctx->input_filename);
printf("Output: %s\n", g_media_ctx->output_filename);
printf("Video filter: %s\n", g_media_ctx->video_filter_desc);
// Open input file
if (open_input_file(g_media_ctx) < 0) {
goto cleanup;
}
// Create output file
if (create_output_file(g_media_ctx) < 0) {
goto cleanup;
}
// Initialize filters
if (init_filter_graph(g_media_ctx) < 0) {
goto cleanup;
}
// Start processing
g_media_ctx->running = true;
if (pthread_create(&g_media_ctx->processing_thread, NULL,
processing_thread_func, g_media_ctx) != 0) {
log_error("Failed to create processing thread");
goto cleanup;
}
// Wait for processing to complete
pthread_join(g_media_ctx->processing_thread, NULL);
printf("Media processing completed successfully\n");
printf("Total frames processed: %ld\n", g_media_ctx->processed_frames);
printf("Average processing speed: %.1f fps\n", g_media_ctx->processing_fps);
cleanup:
media_context_destroy(g_media_ctx);
return 0;
}
Build and Testing Framework
#!/bin/bash
# multimedia_build_framework.sh - Comprehensive multimedia development framework
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BUILD_DIR="$SCRIPT_DIR/build"
TEST_DIR="$SCRIPT_DIR/tests"
INSTALL_DIR="$SCRIPT_DIR/install"
echo "=== Advanced Linux Multimedia Programming Build Framework ==="
# Setup environment
setup_environment() {
echo "Setting up multimedia development environment..."
mkdir -p "$BUILD_DIR"
mkdir -p "$TEST_DIR"
mkdir -p "$INSTALL_DIR"
# Install ALSA development libraries
if ! pkg-config --exists alsa; then
echo "Installing ALSA development libraries..."
sudo apt-get update
sudo apt-get install -y libasound2-dev
fi
# Install PulseAudio development libraries
if ! pkg-config --exists libpulse; then
echo "Installing PulseAudio development libraries..."
sudo apt-get install -y libpulse-dev pulseaudio-module-dev
fi
# Install FFmpeg development libraries
if ! pkg-config --exists libavcodec; then
echo "Installing FFmpeg development libraries..."
sudo apt-get install -y libavcodec-dev libavformat-dev libavutil-dev \
libavfilter-dev libswscale-dev libswresample-dev libavdevice-dev
fi
# Install additional audio processing libraries
sudo apt-get install -y libfftw3-dev libsamplerate0-dev libsndfile1-dev \
libjack-jackd2-dev libportaudio2-dev
echo "Environment setup completed"
}
# Build ALSA applications
build_alsa_applications() {
echo "Building ALSA applications..."
cd "$BUILD_DIR"
# Copy source files
cp "$SCRIPT_DIR"/alsa_advanced.c .
# Build ALSA advanced framework
gcc -o alsa_advanced alsa_advanced.c \
$(pkg-config --cflags --libs alsa) \
-lfftw3 -lsamplerate -lsndfile -lm -lpthread -lrt
# Create ALSA test program
cat > alsa_test.c << 'EOF'
#include "alsa_advanced.c"
int main() {
printf("ALSA Advanced Audio Processing Test\n");
printf("==================================\n");
// List available devices
printf("Available ALSA devices:\n");
void **hints;
int err = snd_device_name_hint(-1, "pcm", &hints);
if (err == 0) {
void **n = hints;
while (*n != NULL) {
char *name = snd_device_name_get_hint(*n, "NAME");
char *desc = snd_device_name_get_hint(*n, "DESC");
printf(" %s: %s\n", name ? name : "Unknown", desc ? desc : "No description");
free(name);
free(desc);
n++;
}
snd_device_name_free_hint(hints);
}
return 0;
}
EOF
gcc -o alsa_test alsa_test.c $(pkg-config --cflags --libs alsa)
echo "ALSA applications built successfully"
}
# Build PulseAudio module
build_pulseaudio_module() {
echo "Building PulseAudio module..."
cd "$BUILD_DIR"
# Copy module source
cp "$SCRIPT_DIR"/module_advanced_processor.c .
# Create module Makefile
cat > Makefile.pulse << 'EOF'
CFLAGS = $(shell pkg-config --cflags libpulse) -fPIC -DPIC -DHAVE_CONFIG_H
LDFLAGS = $(shell pkg-config --libs libpulse) -shared -lfftw3
MODULE = module-advanced-processor.so
all: $(MODULE)
$(MODULE): module_advanced_processor.c
$(CC) $(CFLAGS) -o $@ $< $(LDFLAGS)
install: $(MODULE)
sudo cp $(MODULE) $(shell pkg-config --variable=modlibexecdir libpulse)
load:
pactl load-module module-advanced-processor sink_name=advanced_sink
unload:
pactl unload-module module-advanced-processor || true
clean:
rm -f $(MODULE)
.PHONY: all install load unload clean
EOF
# Build module
make -f Makefile.pulse all
echo "PulseAudio module built successfully"
}
# Build FFmpeg applications
build_ffmpeg_applications() {
echo "Building FFmpeg applications..."
cd "$BUILD_DIR"
# Copy source files
cp "$SCRIPT_DIR"/ffmpeg_advanced.c .
# Build FFmpeg framework
gcc -o ffmpeg_advanced ffmpeg_advanced.c \
$(pkg-config --cflags --libs libavcodec libavformat libavutil \
libavfilter libswscale libswresample libavdevice) \
-lm -lpthread
# Create media processing test
cat > media_test.c << 'EOF'
#include <stdio.h>
#include <libavformat/avformat.h>
#include <libavcodec/avcodec.h>
int main(int argc, char *argv[]) {
if (argc < 2) {
printf("Usage: %s <media_file>\n", argv[0]);
return 1;
}
av_register_all();
AVFormatContext *fmt_ctx = NULL;
if (avformat_open_input(&fmt_ctx, argv[1], NULL, NULL) < 0) {
printf("Error opening file\n");
return 1;
}
if (avformat_find_stream_info(fmt_ctx, NULL) < 0) {
printf("Error finding stream info\n");
avformat_close_input(&fmt_ctx);
return 1;
}
printf("Media file analysis:\n");
printf("====================\n");
printf("Format: %s\n", fmt_ctx->iformat->long_name);
printf("Duration: %ld seconds\n", fmt_ctx->duration / AV_TIME_BASE);
printf("Streams: %d\n", fmt_ctx->nb_streams);
for (int i = 0; i < fmt_ctx->nb_streams; i++) {
AVStream *stream = fmt_ctx->streams[i];
AVCodecParameters *codecpar = stream->codecpar;
printf("Stream %d:\n", i);
printf(" Type: %s\n", av_get_media_type_string(codecpar->codec_type));
printf(" Codec: %s\n", avcodec_get_name(codecpar->codec_id));
if (codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
printf(" Resolution: %dx%d\n", codecpar->width, codecpar->height);
printf(" Frame rate: %.2f fps\n", av_q2d(stream->r_frame_rate));
} else if (codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
printf(" Sample rate: %d Hz\n", codecpar->sample_rate);
printf(" Channels: %d\n", codecpar->channels);
}
}
avformat_close_input(&fmt_ctx);
return 0;
}
EOF
gcc -o media_test media_test.c \
$(pkg-config --cflags --libs libavformat libavcodec libavutil)
echo "FFmpeg applications built successfully"
}
# Run audio tests
run_audio_tests() {
echo "Running audio system tests..."
cd "$BUILD_DIR"
# Test ALSA device enumeration
echo "=== ALSA Device Test ==="
if [ -x ./alsa_test ]; then
./alsa_test
else
echo "ALSA test not available"
fi
# Test PulseAudio module (if PulseAudio is running)
echo -e "\n=== PulseAudio Module Test ==="
if systemctl --user is-active --quiet pulseaudio || pgrep -x pulseaudio > /dev/null; then
echo "PulseAudio is running"
if [ -f module-advanced-processor.so ]; then
echo "Loading advanced processor module..."
make -f Makefile.pulse load || echo "Module load failed (expected if already loaded)"
# List loaded modules
echo "Loaded PulseAudio modules:"
pactl list modules short | grep advanced || echo "Advanced processor module not found"
# Cleanup
make -f Makefile.pulse unload || true
fi
else
echo "PulseAudio not running, skipping module test"
fi
# Test JACK (if available)
echo -e "\n=== JACK Audio Test ==="
if command -v jackd &> /dev/null; then
echo "JACK audio system available"
if pgrep -x jackd > /dev/null; then
echo "JACK is running"
jack_lsp || echo "No JACK ports available"
else
echo "JACK not running"
fi
else
echo "JACK not installed"
fi
}
# Run multimedia tests
run_multimedia_tests() {
echo "Running multimedia processing tests..."
cd "$BUILD_DIR"
# Create test media files
echo "Creating test media files..."
# Generate test audio
if command -v ffmpeg &> /dev/null; then
ffmpeg -f lavfi -i "sine=frequency=440:duration=5" -ac 2 test_audio.wav -y 2>/dev/null
# Generate test video
ffmpeg -f lavfi -i "testsrc2=duration=5:size=640x480:rate=25" \
-f lavfi -i "sine=frequency=440:duration=5" \
-c:v libx264 -c:a aac test_video.mp4 -y 2>/dev/null
echo "Test media files created"
# Test media analysis
echo -e "\n=== Media Analysis Test ==="
if [ -x ./media_test ]; then
echo "Analyzing test audio file:"
./media_test test_audio.wav
echo -e "\nAnalyzing test video file:"
./media_test test_video.mp4
fi
# Test advanced processing
echo -e "\n=== Advanced Processing Test ==="
if [ -x ./ffmpeg_advanced ]; then
echo "Processing test video with filters..."
./ffmpeg_advanced test_video.mp4 processed_video.mp4 "scale=320:240,hqdn3d" &
PROC_PID=$!
sleep 10
kill $PROC_PID 2>/dev/null || true
wait $PROC_PID 2>/dev/null || true
if [ -f processed_video.mp4 ]; then
echo "Processed video created successfully"
ls -lh processed_video.mp4
else
echo "Processing test incomplete"
fi
fi
else
echo "FFmpeg not available, skipping media tests"
fi
}
# Performance benchmarking
run_performance_benchmarks() {
echo "Running multimedia performance benchmarks..."
cd "$BUILD_DIR"
# Audio latency test
echo "=== Audio Latency Benchmark ==="
if [ -x ./alsa_advanced ]; then
echo "Testing ALSA real-time performance..."
timeout 10s ./alsa_advanced default default || echo "ALSA test completed"
fi
# Video processing benchmark
echo -e "\n=== Video Processing Benchmark ==="
if command -v ffmpeg &> /dev/null && [ -f test_video.mp4 ]; then
echo "Benchmarking video encoding performance..."
time ffmpeg -i test_video.mp4 -c:v libx264 -preset ultrafast \
-f null - 2>/dev/null || true
echo "Benchmarking video filtering performance..."
time ffmpeg -i test_video.mp4 -vf "scale=1280:720,hqdn3d" \
-f null - 2>/dev/null || true
fi
# Audio processing benchmark
echo -e "\n=== Audio Processing Benchmark ==="
if command -v ffmpeg &> /dev/null && [ -f test_audio.wav ]; then
echo "Benchmarking audio processing..."
time ffmpeg -i test_audio.wav -af "equalizer=f=1000:width_type=h:width=200:g=10" \
-f null - 2>/dev/null || true
fi
}
# Generate comprehensive report
generate_report() {
local report_file="$BUILD_DIR/multimedia_report.html"
echo "Generating multimedia development report..."
cat > "$report_file" << 'EOF'
<!DOCTYPE html>
<html>
<head>
<title>Linux Multimedia Programming Report</title>
<style>
body { font-family: Arial, sans-serif; margin: 20px; }
.section { margin: 20px 0; padding: 15px; border: 1px solid #ddd; }
.metric { margin: 10px 0; }
table { border-collapse: collapse; width: 100%; }
th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
th { background-color: #f2f2f2; }
.good { color: green; }
.warning { color: orange; }
.poor { color: red; }
pre { background: #f5f5f5; padding: 10px; overflow-x: auto; }
</style>
</head>
<body>
<h1>Advanced Linux Multimedia Programming Report</h1>
<div class="section">
<h2>Development Environment</h2>
<div class="metric">Generated: <script>document.write(new Date())</script></div>
<div class="metric">Build Directory: BUILD_DIR_PLACEHOLDER</div>
<div class="metric">Audio Subsystems: ALSA, PulseAudio, JACK</div>
<div class="metric">Multimedia Framework: FFmpeg</div>
</div>
<div class="section">
<h2>Audio System Status</h2>
<table>
<tr>
<th>Component</th>
<th>Status</th>
<th>Version</th>
<th>Notes</th>
</tr>
<tr>
<td>ALSA</td>
<td id="alsa-status">-</td>
<td id="alsa-version">-</td>
<td>Low-level audio interface</td>
</tr>
<tr>
<td>PulseAudio</td>
<td id="pulse-status">-</td>
<td id="pulse-version">-</td>
<td>User-space audio server</td>
</tr>
<tr>
<td>JACK</td>
<td id="jack-status">-</td>
<td id="jack-version">-</td>
<td>Professional audio server</td>
</tr>
<tr>
<td>FFmpeg</td>
<td id="ffmpeg-status">-</td>
<td id="ffmpeg-version">-</td>
<td>Multimedia framework</td>
</tr>
</table>
</div>
<div class="section">
<h2>Built Applications</h2>
<ul>
<li>ALSA Advanced Framework - Real-time audio processing</li>
<li>PulseAudio Advanced Module - DSP effects processing</li>
<li>FFmpeg Advanced Framework - Media processing pipeline</li>
<li>Media Analysis Tools - Format and codec inspection</li>
</ul>
</div>
<div class="section">
<h2>Performance Metrics</h2>
<div id="performance-metrics">
<p>Audio latency, video processing speed, and codec performance results...</p>
</div>
</div>
<div class="section">
<h2>Development Guidelines</h2>
<ul>
<li>Use ALSA for low-latency real-time audio applications</li>
<li>Implement PulseAudio modules for system-wide audio effects</li>
<li>Leverage FFmpeg for multimedia format support</li>
<li>Consider JACK for professional audio workflows</li>
<li>Profile audio code for real-time constraints</li>
<li>Test across different hardware configurations</li>
</ul>
</div>
</body>
</html>
EOF
# Replace placeholder with actual directory
sed -i "s|BUILD_DIR_PLACEHOLDER|$BUILD_DIR|g" "$report_file"
echo "Report generated: $report_file"
echo "Open in browser: file://$report_file"
}
# Cleanup function
cleanup() {
echo "Cleaning up multimedia build environment..."
cd "$BUILD_DIR"
# Unload PulseAudio module
make -f Makefile.pulse unload 2>/dev/null || true
# Remove test files
rm -f test_audio.wav test_video.mp4 processed_video.mp4
echo "Cleanup completed"
}
# Main execution
main() {
case "${1:-help}" in
setup)
setup_environment
;;
build-alsa)
build_alsa_applications
;;
build-pulse)
build_pulseaudio_module
;;
build-ffmpeg)
build_ffmpeg_applications
;;
build-all)
setup_environment
build_alsa_applications
build_pulseaudio_module
build_ffmpeg_applications
;;
test-audio)
run_audio_tests
;;
test-multimedia)
run_multimedia_tests
;;
benchmark)
run_performance_benchmarks
;;
report)
generate_report
;;
all)
setup_environment
build_alsa_applications
build_pulseaudio_module
build_ffmpeg_applications
run_audio_tests
run_multimedia_tests
run_performance_benchmarks
generate_report
;;
cleanup)
cleanup
;;
*)
echo "Usage: $0 {setup|build-alsa|build-pulse|build-ffmpeg|build-all|test-audio|test-multimedia|benchmark|report|all|cleanup}"
echo ""
echo "Commands:"
echo " setup - Setup development environment"
echo " build-alsa - Build ALSA applications"
echo " build-pulse - Build PulseAudio module"
echo " build-ffmpeg - Build FFmpeg applications"
echo " build-all - Build all applications"
echo " test-audio - Test audio subsystems"
echo " test-multimedia - Test multimedia processing"
echo " benchmark - Run performance benchmarks"
echo " report - Generate development report"
echo " all - Run complete workflow"
echo " cleanup - Clean up build environment"
;;
esac
}
# Handle signals for cleanup
trap cleanup EXIT INT TERM
main "$@"
This comprehensive Linux audio and multimedia programming guide demonstrates:
- Advanced ALSA programming with real-time audio processing and DSP effects
- Custom PulseAudio module development with professional audio processing
- Complete FFmpeg integration for multimedia processing pipelines
- Production-ready build and testing frameworks for multimedia applications
The implementations showcase real-world multimedia programming techniques used in professional audio software and media processing applications.
<function_calls> [{“content”: “Spread existing blog posts across multiple weeks with proper dates”, “status”: “completed”, “priority”: “high”, “id”: “1”}, {“content”: “Create 20 additional advanced Linux/systems programming blog posts”, “status”: “completed”, “priority”: “high”, “id”: “2”}, {“content”: “Create 100 more advanced Linux/systems programming blog posts”, “status”: “in_progress”, “priority”: “high”, “id”: “3”}]