Azure Pronunciation Assessment SDK return wrong result compare with api call

Question:

I’m using azure speech sdk to do pronunciation assessment, it works fine when i used api provide by azure, but when i use speech sdk the result is not correct. I follow the sample from cognitive services speech sdk

Here is the code that i used for sdk

    def speech_recognition_with_pull_stream(self):
    class WavFileReaderCallback(speechsdk.audio.PullAudioInputStreamCallback):
        def __init__(self, filename: str):
            super().__init__()
            self._file_h = wave.open(filename, mode=None)

            self.sample_width = self._file_h.getsampwidth()

            assert self._file_h.getnchannels() == 1
            assert self._file_h.getsampwidth() == 2
            # assert self._file_h.getframerate() == 16000  #comment this line because every .wav file read is 48000
            assert self._file_h.getcomptype() == 'NONE'

        def read(self, buffer: memoryview) -> int:
            size = buffer.nbytes
            print(size)
            print(len(buffer))
            frames = self._file_h.readframes(len(buffer) // self.sample_width)

            buffer[:len(frames)] = frames

            return len(frames)

        def close(self):
            self._file_h.close()

    speech_key = os.getenv('AZURE_SUBSCRIPTION_KEY')
    service_region = os.getenv('AZURE_REGION')
    speech_config = speechsdk.SpeechConfig(subscription=speech_key, region=service_region)

    # specify the audio format
    wave_format = speechsdk.audio.AudioStreamFormat(samples_per_second=16000, bits_per_sample=16, channels=1)

    # setup the audio stream
    callback = WavFileReaderCallback('/Users/146072/Downloads/58638f26-ed07-40b7-8672-1948c814bd69.wav')
    stream = speechsdk.audio.PullAudioInputStream(callback, wave_format)
    audio_config = speechsdk.audio.AudioConfig(stream=stream)

    # instantiate the speech recognizer with pull stream input
    speech_recognizer = speechsdk.SpeechRecognizer(speech_config=speech_config, audio_config=audio_config, language='en-US')

    reference_text = 'We had a great time taking a long walk outside in the morning'
    pronunciation_assessment_config = speechsdk.PronunciationAssessmentConfig(
        reference_text=reference_text,
        grading_system=PronunciationAssessmentGradingSystem.HundredMark,
        granularity=PronunciationAssessmentGranularity.Word,
    )
    pronunciation_assessment_config.phoneme_alphabet = "IPA"
    pronunciation_assessment_config.apply_to(speech_recognizer)
    speech_recognition_result = speech_recognizer.recognize_once()
    print(speech_recognition_result.text)

    # The pronunciation assessment result as a Speech SDK object
    pronunciation_assessment_result = speechsdk.PronunciationAssessmentResult(speech_recognition_result)
    print(pronunciation_assessment_result)

    # The pronunciation assessment result as a JSON string
    pronunciation_assessment_result_json = speech_recognition_result.properties.get(
        speechsdk.PropertyId.SpeechServiceResponse_JsonResult
    )
    print(pronunciation_assessment_result_json)

    return json.loads(pronunciation_assessment_result_json)

and here is the result from sdk

"PronunciationAssessment": {
    "AccuracyScore": 26,
    "FluencyScore": 9,
    "CompletenessScore": 46,
    "PronScore": 19.8
  },

and here is the code for api call

    def ackaud(self):
    #    f.save(audio)
    # print('file uploaded successfully')

    # a generator which reads audio data chunk by chunk
    # the audio_source can be any audio input stream which provides read() method, e.g. audio file, microphone, memory stream, etc.
    def get_chunk(audio_source, chunk_size=1024):
        while True:
            # time.sleep(chunk_size / 32000) # to simulate human speaking rate
            chunk = audio_source.read(chunk_size)
            if not chunk:
                # global uploadFinishTime
                # uploadFinishTime = time.time()
                break
            yield chunk

    # build pronunciation assessment parameters
    referenceText = 'We had a great time taking a long walk outside in the morning. '

    pronAssessmentParamsJson = "{"ReferenceText":"%s","GradingSystem":"HundredMark","Dimension":"Comprehensive","EnableMiscue":"True"}" % referenceText
    pronAssessmentParamsBase64 = base64.b64encode(bytes(pronAssessmentParamsJson, 'utf-8'))
    pronAssessmentParams = str(pronAssessmentParamsBase64, "utf-8")

    subscription_key = os.getenv('AZURE_SUBSCRIPTION_KEY')
    region = os.getenv('AZURE_REGION')

    # build request
    url = "https://%s.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1?language=%s&usePipelineVersion=0" % (
    region, 'en-US')
    headers = {'Accept': 'application/json;text/xml',
               'Connection': 'Keep-Alive',
               'Content-Type': 'audio/wav; codecs=audio/pcm; samplerate=16000',
               'Ocp-Apim-Subscription-Key': subscription_key,
               'Pronunciation-Assessment': pronAssessmentParams,
               'Transfer-Encoding': 'chunked',
               'Expect': '100-continue'}

    audioFile = open('/Users/146072/Downloads/58638f26-ed07-40b7-8672-1948c814bd69.wav', 'rb')
    # audioFile = f
    # send request with chunked data
    response = requests.post(url=url, data=get_chunk(audioFile), headers=headers)
    # getResponseTime = time.time()
    audioFile.close()

    # latency = getResponseTime - uploadFinishTime
    # print("Latency = %sms" % int(latency * 1000))

    return response.json()

and here is the result from api

"AccuracyScore": 100,
"FluencyScore": 100,
"CompletenessScore": 100,
"PronScore": 100,

Am i doing anything wrong in the setup? Thanks a lot.

Asked By: TRose

||

Answers:

Install the latest Speech SDK 1.26.0 as REST API uses version 3.1 that is generally available.

Here is the document to install the speech SDK.

Answered By: Ram

After playing around with samples_per_second the code seem to work correct.

Answered By: TRose