Stimme in einer mobilen App: Lernen, Bildschirme anzurufen und Formulare ohne Hände auszufüllen

Wie kann eine Sprachschnittstelle schnell und nahtlos in Ihre mobile Anwendung eingebettet werden? Und wie kann man einem Assistenten alles beibringen, was er kann? Das letzte Mal haben wir die OpenSorsnoe-Lifestyle-Anwendung Habitica verwendet und gezeigt, wie der Assistent und das grundlegende Sprach-Skript "out of the box" hinzugefügt werden (Wettervorhersage und Uhrzeit aktualisieren). Und jetzt gehen wir zu einem fortgeschritteneren Stadium über - wir werden lernen, wie man bestimmte Bildschirme per Sprache anruft, komplexe Anforderungen mit NLU stellt und Formulare mit einer Stimme in der Anwendung ausfüllt.

(Lesen Sie den ersten Teil des Tutorials)

, Habitica – : ,     . , , , , .

– . , , . AndroidManifest . PrefsActivity, , FixCharacterValuesActivity, , , , FullProfileActivity AboutActivity.

, , CustomSkill. -, , , response.action “changeView”. response.intent , – . :

class ChangeViewSkill(private val context: Context): CustomSkill<AimyboxRequest, AimyboxResponse> {

    override fun canHandle(response: AimyboxResponse) = response.action == "changeView"

    override suspend fun onResponse(
            response: AimyboxResponse,
            aimybox: Aimybox,
            defaultHandler: suspend (Response) -> Unit
    ) {
        val intent = when (response.intent) {
            "settings" -> Intent(context, PrefsActivity::class.java)
            "characteristics" -> Intent(context, FixCharacterValuesActivity::class.java)//
            "profile" -> Intent(context, FullProfileActivity::class.java)//
            "about" -> Intent(context, AboutActivity::class.java)
            else -> Intent(context, MainActivity::class.java)
        }
        intent.setFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
        aimybox.standby()
        context.startActivity(intent)
    }
}

:

 val dialogApi = AimyboxDialogApi(
                "YOUR KEY HERE", unitId,
            customSkills = linkedSetOf(ChangeView()))

 JAICF ( Just AI Kotlin).

https://github.com/just-ai/jaicf-jaicp-caila-template.

, JAICP (Just AI Conversational Platform) c Aimybox (SDK ), – connections. , . AimyboxConnection.

package com.justai.jaicf.template.connections
import com.justai.jaicf.channel.http.httpBotRouting
import com.justai.jaicf.channel.aimybox.AimyboxChannel
import io.ktor.routing.routing
import io.ktor.server.engine.embeddedServer
import io.ktor.server.netty.Netty
import com.justai.jaicf.template.templateBot

fun main() {
    embeddedServer(Netty, System.getenv("PORT")?.toInt() ?: 8080) {
        routing {
            httpBotRouting("/" to AimyboxChannel(templateBot))
        }
    }.start(wait = true)
}

, NLU-, NLU- Caila – app.jaicp.com, API conf/jaicp.properties. , app.jaicp.com.



NLU- – , , NLU.

. , . , , DATA , (settings, characteristics, .. ).

:

, . . , , , , views . .

JAICF.

, , - «» . .

:

, catchAll – , , . changeView, activators JAICP , actions – , Aimybox , .

views , Caila, action , , Aimybox , , . «». - .

        state("changeView") {
            activators {
                intent("changeView")
            }
            action {
                reactions.say("..." )
                var slot = ""
                activator.caila?.run {slot = slots["views"].toString()}
                reactions.aimybox?.response?.action = "changeView"
                reactions.aimybox?.response?.intent = slot

            }
        }

skills .

. ngrok, heroku. app.aimybox.com, , Aimylogic webhook URL. : , .

, , Try in Action.

, – , .

. . , .

! .

, , , - (- ), .

. response.action == "createTask", , response.intent. 

, , , , , TaskFormActivity, . .

class CreateTaskSkill(private val context: Context): CustomSkill<AimyboxRequest, AimyboxResponse> {


    override fun canHandle(response: AimyboxResponse) = response.action == "createTask"

    override suspend fun onResponse(
            response: AimyboxResponse,
            aimybox: Aimybox,
            defaultHandler: suspend (Response) -> Unit
    ) {
        val intent = Intent(context, TaskFormActivity::class.java)
        val additionalData = HashMap<String, Any>()
        val type = response.intent
        additionalData["viewed task type"] = when (type) {
            "habit" -> Task.TYPE_HABIT
            "daily" -> Task.TYPE_DAILY
            "todo" -> Task.TYPE_TODO
            "reward" -> Task.TYPE_REWARD
            else -> ""
        }

( ) , . .

response.data, , .

. onCreate TaskFormActivity.

// Inserted code for voice activation
        textEditText.setText(bundle.getString("activity_name")) // presetting task name
        notesEditText.setText(bundle.getString("activity_description")) //presetting task description
        if (bundle.getBoolean("sentiment")) {  // presetting task sentiment
            habitScoringButtons.isPositive = true
            habitScoringButtons.isNegative = false
        } else {
            habitScoringButtons.isNegative = true
            habitScoringButtons.isPositive = false
        }
        when (bundle.getString("activity_difficulty").toString()) { // presetting task difficulty
            "trivial" -> taskDifficultyButtons.selectedDifficulty = 0.1f
            "easy" -> taskDifficultyButtons.selectedDifficulty = 1f
            "medium" -> taskDifficultyButtons.selectedDifficulty = 1.5f
            "hard" -> taskDifficultyButtons.selectedDifficulty = 2f
            else -> taskDifficultyButtons.selectedDifficulty = 1f
        }

JAICF Caila.

Caila: , ( , Pattern ).

data , – habit, pattern .

, Name Description, , . .

:

, task_type . , – , , , .

, (, ). , .

, . 

@ – , “ – .

JAICF.

state("createTask") {
            activators {
                intent("createTask")
            }
            action {
                val taskType = activator.getCailaSlot("taskType").asJsonLiteralOr("")
                reactions.say("...")
                reactions.aimybox?.response?.action = "createTask"
                reactions.aimybox?.response?.intent = taskType.content
                reactions.aimybox?.response?.run {
                    data["taskName"] = activator.getCailaSlot("taskName").asJsonLiteralOr("")
                    data["taskDescription"] = activator.getCailaSlot("taskDescription").asJsonLiteralOr("")
                    data["taskSentiment"] = activator.getCailaSlotBool("taskSentiment").asJsonLiteralOr(true)
                    data["taskDifficulty"] = activator.getCailaSlot("taskDifficulty").asJsonLiteralOr("easy")
                }
            }
        } 
private fun ActivatorContext.getCailaRequiredSlot(k: String): String =
    getCailaSlot(k) ?: error("Missing Caila slot for key: $k")

private fun ActivatorContext.getCailaSlot(k: String): String? =
    caila?.slots?.get(k)

private fun ActivatorContext.getCailaSlotBool(k: String): Boolean? =
    caila?.slots?.get(k)?.toBoolean()

private fun String?.asJsonLiteralOr(other: String) = this?.let { JsonLiteral(this) } ?: JsonLiteral(other)
private fun Boolean?.asJsonLiteralOr(other: Boolean) = this?.let { JsonLiteral(this) } ?: JsonLiteral(other)

, intent, data, action, Aimybox , .

, ! :

, – , . !

JAICF.

Aimybox.