Use prebuilt Azure AI Translator in Fabric with REST API and SynapseML (preview)

# Get workload endpoints and access token

from synapse.ml.mlflow import get_mlflow_env_config
import json

mlflow_env_configs = get_mlflow_env_config()
access_token = access_token = mlflow_env_configs.driver_aad_token
prebuilt_AI_base_host = mlflow_env_configs.workload_endpoint + "cognitive/texttranslation/"
print("Workload endpoint for AI service: \n" + prebuilt_AI_base_host)

# Make a RESTful request to AI service

post_headers = {
    "Content-Type" : "application/json",
    "Authorization" : "Bearer {}".format(access_token),
}

def printresponse(response):
    print(f"HTTP {response.status_code}")
    if response.status_code == 200:
        try:
            result = response.json()
            print(json.dumps(result, indent=2, ensure_ascii=False))
        except:
            print(f"pasre error {response.content}")
    else:
        print(f"error message: {response.content}")

import synapse.ml.core
from synapse.ml.cognitive.translate import *
from pyspark.sql.functions import col, flatten

The core operation of the Translator service is to translate text.

import requests
import uuid

service_url = prebuilt_AI_base_host + "translate?api-version=3.0&to=fr"
post_body = [{'Text':'Hello, friend.'}]

post_headers["x-ms-workload-resource-moniker"] = str(uuid.uuid1())
response = requests.post(service_url, json=post_body, headers=post_headers)

# Output all information of the request process
printresponse(response)

Output

    HTTP 200
    [
      {
        "detectedLanguage": {
          "language": "en",
          "score": 1.0
        },
        "translations": [
          {
            "text": "Bonjour cher ami.",
            "to": "fr"
          }
        ]
      }
    ]

The core operation of the Translator service is to translate text.

df = spark.createDataFrame([
  (["Hello, what is your name?", "Bye"],)
], ["text",])

translate = (Translate()
    .setTextCol("text")
    .setToLanguage(["zh-Hans", "fr"])
    .setOutputCol("translation")
    .setConcurrency(5))

result = translate.transform(df)\
        .withColumn("translation", flatten(col("translation.translations")))\
        .withColumn("translation", col("translation.text"))

display(result.select("text", "translation"))

Transliteration is the process of converting a word or phrase from the script (alphabet) of one language to another based on phonetic similarity.

service_url = prebuilt_AI_base_host + "transliterate?api-version=3.0&language=ja&fromScript=Jpan&toScript=Latn"
post_body = [
    {"Text":"こんにちは"},
    {"Text":"さようなら"}
]

post_headers["x-ms-workload-resource-moniker"] = str(uuid.uuid1())
response = requests.post(service_url, json=post_body, headers=post_headers)

# Output all information of the request process
printresponse(response)

Output

    HTTP 200
    [
      {
        "text": "Kon'nichiwa​",
        "script": "Latn"
      },
      {
        "text": "sayonara",
        "script": "Latn"
      }
    ]

Transliteration is the process of converting a word or phrase from the script (alphabet) of one language to another based on phonetic similarity.

transliterateDf =  spark.createDataFrame([
  (["こんにちは", "さようなら"],)
], ["text",])

transliterate = (Transliterate()
    .setLanguage("ja")
    .setFromScript("Jpan")
    .setToScript("Latn")
    .setTextCol("text")
    .setOutputCol("result"))

result = transliterate.transform(transliterateDf)\
        .withColumn("script", col("result.script"))

display(result.select("text", "script"))

Gets a list of languages supported by the operations of Translator.

service_url = prebuilt_AI_base_host + "languages?api-version=3.0"

post_headers["x-ms-workload-resource-moniker"] = str(uuid.uuid1())
response = requests.get(service_url, headers=post_headers)

# Output all information of the request process
printresponse(response)

No steps for SynapseML in this section.