No audio when using SpeechSDK in pcf control (canvas app)

Joël Simons 0 Reputation points
2024-07-03T13:38:13.5566667+00:00

I made a pcf control which uses the speechsdk to synthesize text to speech. This is working when I run "npm start watch" to test this. When publishing this pcf control and use it in a canvas powerapp I cannot hear the synthisized text.

What can be wrong?

    private onStartButtonClick(e: Event) {
        this.displayTempResult.value = this.messageToSay || "";

        const player = new SpeechSDK.SpeakerAudioDestination();

        if (player) {
            console.log('Player is initialized');
        } else {
            console.error('Player is NOT initialized');
        }

        player.onAudioStart = () => {

            console.log('playback started');

        };

        player.onAudioEnd = () => {

            console.log('playback finished');

        };

        console.log('Before AudioConfig');
        const audioConfig = SpeechSDK.AudioConfig.fromSpeakerOutput(player);

        console.log('Before SpeechConfig');
        const speechConfig = SpeechSDK.SpeechConfig.fromSubscription(this._context.parameters.SubscriptionKey.raw || "", this._context.parameters.AzureRegion.raw || "westeurope");

        speechConfig.speechSynthesisLanguage = "nl-nl";
        speechConfig.speechSynthesisVoiceName = "de-DE-FlorianMultilingualNeural";
        speechConfig.speechSynthesisOutputFormat = SpeechSDK.SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3;

        console.log('Before new SpeechSynthesizer');
        const ss = new SpeechSDK.SpeechSynthesizer(speechConfig, audioConfig);

        ss.synthesizing = (s, e) => {

            console.log('Synthesizing...');
            console.log(e);

            // Handle synthesizing event
        };

        ss.synthesisStarted = (s, e) => {

            console.log('Synthesis started');
            console.log(e);

            // Handle synthesis started event
        };

        ss.synthesisCompleted = (s, e) => {

            console.log('Synthesis completed');
            console.log(e);

            // Handle synthesis completed event
        };

        ss.SynthesisCanceled = (s, e) => {

            const cancellationDetails = SpeechSDK.CancellationDetails.fromResult(e.result);

            let str = "(cancel) Reason: " + SpeechSDK.CancellationReason[cancellationDetails.reason];

            if (cancellationDetails.reason === SpeechSDK.CancellationReason.Error) {

                str += ": " + e.result.errorDetails;

            }

            console.log('Synthesis canceled');
            console.log(e);

            // Handle synthesis canceled event
        };

        console.log('Before SpeakTextAsync');

        if (ss) {
            console.log('Synthesizer is initialized and want to say: ' + this.messageToSay);
            ss.speakTextAsync(
                this.messageToSay,
                result => {
                    if (result.reason === SpeechSDK.ResultReason.SynthesizingAudioCompleted) {
                        console.log('Result --> Synthesis finished.');
                    } else {
                        console.error('Result --> Speech synthesis canceled, ' + result.errorDetails);
                    }
                },
                error => {
                    console.error(error);
                }
            );
        } else {
            console.error('Synthesizer is NOT initialized');
        }
    }

Azure AI Speech
Azure AI Speech
An Azure service that integrates speech processing into apps and services.
1,519 questions
{count} votes