Google Gemini's Multimodal Live API with C# .Net

Install-Package Google_GenerativeAI.Live
Install-Package NAudio

// Create the client with the desired configuration
var client = new MultiModalLiveClient(
    platformAdapter: new GoogleAIPlatformAdapter(), 
    modelName: "gemini-1.5-flash-exp", 
    generationConfig: new GenerationConfig 
    { 
        ResponseModalities = { Modality.TEXT, Modality.AUDIO } 
    }, 
    safetySettings: null, 
    systemInstruction: "You are a helpful assistant."
);

// Subscribe to events
client.Connected += (s, e) => Console.WriteLine("Connected to Gemini!");
client.TextChunkReceived += (s, e) => Console.WriteLine($"Gemini: {e.TextChunk}");
client.AudioChunkReceived += (s, e) => Console.WriteLine($"Audio received: {e.Buffer.Length} bytes");

// Connect and send messages
await client.ConnectAsync();
await client.SentTextAsync("Hello, Gemini!");

// Setup real-time audio streaming Using NAudio
_waveIn = new WaveInEvent { WaveFormat = new WaveFormat(16000, 1) };
_waveIn.DataAvailable += async (s, e) => {
    // Send audio chunks as they arrive
    if (_isListening)
    {
        await _client.SendAudioAsync(
            audioData: e.Buffer,
            audioContentType: "audio/pcm; rate=16000"
        );
    }
};

// Start listening and streaming
public void StartRealTimeListening()
{
    _isListening = true;
    _waveIn.StartRecording();
    Console.WriteLine("Streaming audio to Gemini in real-time...");
}

// Stop listening
public void StopListening()
{
    _isListening = false;
    _waveIn.StopRecording();
}

private BufferedWaveProvider _bufferedWaveProvider;
private WaveOutEvent _waveOut;

// Setup audio playback system
private void InitializeAudioPlayback()
{
    var waveFormat = new WaveFormat(16000, 16, 1);
    _bufferedWaveProvider = new BufferedWaveProvider(waveFormat);
    _waveOut = new WaveOutEvent();
    _waveOut.Init(_bufferedWaveProvider);
}

// Handle incoming audio chunks for immediate playback
client.AudioChunkReceived += (s, e) => {
    // Add received audio to buffer
    _bufferedWaveProvider.AddSamples(e.Buffer, 0, e.Buffer.Length);

    // Start playback if not already playing
    if (_waveOut.PlaybackState != PlaybackState.Playing)
    {
        _waveOut.Play();
    }
};

// Handle playback completion
client.AudioReceiveCompleted += (s, e) => {
    Console.WriteLine("Audio response complete");
    // Optional: wait for buffered audio to finish playing
};

// Core connection events
client.Connected += (s, e) => Console.WriteLine("Connected!");
client.Disconnected += (s, e) => Console.WriteLine("Disconnected");
client.ErrorOccurred += (s, e) => Console.WriteLine($"Error: {e.Exception.Message}");

// Data events
client.TextChunkReceived += (s, e) => ProcessTextChunk(e.TextChunk);
client.AudioChunkReceived += (s, e) => StoreAudioChunk(e.Buffer);
client.AudioReceiveCompleted += (s, e) => PlayStoredAudio();

// Generation config
var generationConfig = new GenerationConfig
{
    Temperature = 0.7f,
    TopP = 0.95f,
    ResponseModalities = { Modality.TEXT, Modality.AUDIO }
};

// Safety settings
var safetySettings = new List<SafetySetting>
{
    new SafetySetting
    {
        Category = HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
        Threshold = HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE
    }
};

// System instructions
var systemInstruction = "You are a helpful technical support assistant.";

public class SimpleVoiceAssistant
{
    private MultiModalLiveClient _client;
    private WaveInEvent _waveIn;
    private BufferedWaveProvider _bufferedWaveProvider;
    private WaveOutEvent _waveOut;
    private bool _isListening = false;

    public async Task InitializeAsync()
    {
        // Initialize client
        _client = new MultiModalLiveClient(
            platformAdapter: new GoogleAIPlatformAdapter(), 
            modelName: "gemini-1.5-flash-exp", 
            generationConfig: new GenerationConfig 
            { 
                ResponseModalities = { Modality.TEXT, Modality.AUDIO } 
            }, 
            safetySettings: null, 
            systemInstruction: "You are a helpful voice assistant."
        );

        // Set up events
        SetupEvents();

        // Initialize audio
        SetupAudio();

        // Connect
        await _client.ConnectAsync();
    }

    private void SetupEvents()
    {
        _client.TextChunkReceived += (s, e) => Console.WriteLine($"Gemini: {e.TextChunk}");
        _client.AudioChunkReceived += HandleAudioChunk;
    }

    private void SetupAudio()
    {
        // Input setup
        _waveIn = new WaveInEvent { WaveFormat = new WaveFormat(16000, 1) };
        _waveIn.DataAvailable += HandleMicrophoneData;

        // Output setup
        var waveFormat = new WaveFormat(16000, 16, 1);
        _bufferedWaveProvider = new BufferedWaveProvider(waveFormat);
        _waveOut = new WaveOutEvent();
        _waveOut.Init(_bufferedWaveProvider);
    }

    private async void HandleMicrophoneData(object sender, WaveInEventArgs e)
    {
        if (_isListening)
        {
            await _client.SendAudioAsync(e.Buffer, "audio/pcm; rate=16000");
        }
    }

    private void HandleAudioChunk(object sender, AudioChunkEventArgs e)
    {
        _bufferedWaveProvider.AddSamples(e.Buffer, 0, e.Buffer.Length);

        if (_waveOut.PlaybackState != PlaybackState.Playing)
        {
            _waveOut.Play();
        }
    }

    public void StartListening()
    {
        _isListening = true;
        _waveIn.StartRecording();
    }

    public void StopListening()
    {
        _isListening = false;
        _waveIn.StopRecording();
    }
}