$(call inherit-product, vendor/rockchip/common/rkllm_qwen25_3b/rkllm_qwen25_3b.mk)

# RKLLM Qwen2.5-3B service integration for RK3576 / Android 14.
# Inherited by device/rockchip/rk3576/device.mk.

RKLLM_QWEN25_DIR := vendor/rockchip/common/rkllm_qwen25_3b

PRODUCT_COPY_FILES += \
    $(RKLLM_QWEN25_DIR)/prebuilt/arm64-v8a/librkllmrt.so:$(TARGET_COPY_OUT_VENDOR)/lib64/librkllmrt.so \
    $(RKLLM_QWEN25_DIR)/prebuilt/arm64-v8a/libomp.so:$(TARGET_COPY_OUT_VENDOR)/lib64/libomp.so \
    $(RKLLM_QWEN25_DIR)/prebuilt/arm64-v8a/llm_demo:$(TARGET_COPY_OUT_VENDOR)/bin/llm_demo \
    $(RKLLM_QWEN25_DIR)/tools/fix_freq_rk3576.sh:$(TARGET_COPY_OUT_VENDOR)/bin/fix_freq_rk3576.sh \
    $(RKLLM_QWEN25_DIR)/init.rkllm_server.rc:$(TARGET_COPY_OUT_VENDOR)/etc/init/init.rkllm_server.rc

PRODUCT_VENDOR_PROPERTIES += \
    persist.vendor.rkllm.server.enable=0 \
    vendor.rkllm.model_path=/data/vendor/rkllm/Qwen2.5-3B-Instruct_W8A8_RK3576.rkllm

# The service resolves librkllmrt.so with dlopen(); the prebuilt libraries
# above are installed by PRODUCT_COPY_FILES, not linked at build time.
PRODUCT_PACKAGES += rkllm_server

# The model is about 3.6 GiB and must NOT be copied into system/vendor images.
# Deploy it to /data/vendor/rkllm/ at first boot or via OTA instead.
BOARD_VENDORIMAGE_PARTITION_SIZE := 402653184
BOARD_SEPOLICY_DIRS += $(RKLLM_QWEN25_DIR)/sepolicy_vendor

# RKLLM OpenAI-compatible HTTP service.
# Disabled by default. Enable it with:
#   setprop persist.vendor.rkllm.server.enable 1

service rkllm_server /vendor/bin/rkllm_server \
    --model /data/vendor/rkllm/Qwen2.5-3B-Instruct_W8A8_RK3576.rkllm \
    --library /vendor/lib64/librkllmrt.so \
    --bind 127.0.0.1 --port 8080
    class main
    user system
    group system graphics inet
    seclabel u:r:rkllm_server:s0
    disabled

on property:persist.vendor.rkllm.server.enable=1
    start rkllm_server

on property:persist.vendor.rkllm.server.enable=0
    stop rkllm_server

adb root
# 模型放到 /data/vendor/rkllm/，与 mk 里的 vendor.rkllm.model_path 保持一致
adb push Qwen2.5-3B-Instruct_W8A8_RK3576.rkllm \
    /data/vendor/rkllm/Qwen2.5-3B-Instruct_W8A8_RK3576.rkllm

setprop persist.vendor.rkllm.server.enable 1

git checkout QY-R3576-DEL
mkdir -p vendor/rockchip/common/rkllm_qwen25_3b/prebuilt/model
mv out/rkllm_model_backup/Qwen2.5-3B-Instruct_W8A8_RK3576.rkllm \
   vendor/rockchip/common/rkllm_qwen25_3b/prebuilt/model/

int RunServer(rkllm::Engine* engine, const std::string& bind_ip, int port,
              const std::string& model) {
    const int listen_fd = socket(AF_INET, SOCK_STREAM, 0);
    int reuse = 1;
    setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse));

    struct sockaddr_in address = {};
    address.sin_family = AF_INET;
    address.sin_port = htons(static_cast<uint16_t>(port));
    if (inet_pton(AF_INET, bind_ip.c_str(), &address.sin_addr) != 1) {
        ALOGE("invalid bind address: %s", bind_ip.c_str());
        close(listen_fd);
        return 1;
    }
    if (bind(listen_fd, reinterpret_cast<struct sockaddr*>(&address), sizeof(address)) < 0) {
        ALOGE("bind(%s:%d) failed: %s", bind_ip.c_str(), port, strerror(errno));
        close(listen_fd);
        return 1;
    }
    if (listen(listen_fd, 8) < 0) {
        ALOGE("listen() failed: %s", strerror(errno));
        close(listen_fd);
        return 1;
    }

    while (g_stop == 0) {
        fd_set fds;
        FD_ZERO(&fds);
        FD_SET(listen_fd, &fds);
        struct timeval timeout = {1, 0};   // 每秒醒一次，好响应退出信号
        const int ready = select(listen_fd + 1, &fds, nullptr, nullptr, &timeout);
        if (ready <= 0) {
            continue;
        }
        const int client_fd = accept(listen_fd, nullptr, nullptr);
        if (client_fd < 0) {
            continue;
        }
        HandleClient(client_fd, engine, model);
        close(client_fd);
    }
    close(listen_fd);
    return 0;
}

void HandleClient(int fd, rkllm::Engine* engine, const std::string& model) {
    struct timeval receive_timeout = {10, 0};  // 读请求 10 秒没数据就放弃
    struct timeval send_timeout = {60, 0};    // 单次 send() 阻塞超 60 秒判失败
    setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &receive_timeout, sizeof(receive_timeout));
    setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &send_timeout, sizeof(send_timeout));

    HttpRequest request;
    if (!ReadHttpRequest(fd, &request)) {
        return;
    }
    if (request.method == "GET" && request.path == "/v1/models") {
        SendJson(fd, 200, "OK", BuildModels(model));
    } else if (request.method == "GET" &&
               (request.path == "/healthz" || request.path == "/")) {
        // 判断设备是否就绪、是否忙碌
        SendJson(fd, 200, "OK", BuildHealth(*engine, model));
    } else if (request.method == "POST" && request.path == "/v1/chat/completions") {
        HandleChatCompletions(fd, request, engine, model);
    } else {
        SendJson(fd, 404, "Not Found", BuildError("Unknown endpoint.", "invalid_request_error"));
    }
}

// 节选：参数映射 + 工具透传
options.top_k               = GetInt(root,   "top_k", 1, 1, 100);
options.top_p               = GetFloat(root, "top_p", 0.9f, 0.0f, 1.0f);
options.temperature         = GetFloat(root, "temperature", 0.8f, 0.0f, 2.0f);
options.max_tokens          = GetInt(root,   "max_tokens", 512, 1, 4096);
options.enable_thinking     = GetBool(root, "enable_thinking", false);

const Json::Value& tools = root["tools"];
if (tools.isArray() && !tools.empty()) {
    Json::StreamWriterBuilder writer;
    writer["indentation"] = "";
    options.tools = Json::writeString(writer, tools);
}

const std::string id = CompletionId();
const bool stream = GetBool(root, "stream", false);

if (!stream) {
    std::string output;
    const rkllm::TextSink sink = [&output](const std::string& text) {
        output += text;
        return true;
    };
    const rkllm::Status status = engine->Generate(prompt, role, options, sink, &error);
    // 此处省略：把 output 组装成 OpenAI 格式的 JSON 响应并 SendJson 回客户端
    return;
}
// ... 以下为流式分支，见 3.6 节

SendSseHeaders(fd);
SendAll(fd, BuildStreamChunk(id, model, "", true, ""));

const rkllm::TextSink sink = [fd, &id, &model](const std::string& text) {
    return SendAll(fd, BuildStreamChunk(id, model, text, false, ""));
};
const rkllm::Status status = engine->Generate(prompt, role, options, sink, &error);
if (status == rkllm::Status::kOk) {
    SendAll(fd, BuildStreamChunk(id, model, "", false, "stop"));
    SendAll(fd, "data: [DONE]\n\n");
} else {
    // 响应头已发出，只能把失败作为 SSE 事件写进流里
    SendAll(fd, "data: " + BuildError(error, "server_error") + "\n\n");
    SendAll(fd, "data: [DONE]\n\n");
}

void Engine::Start() {
    {
        std::lock_guard<std::mutex> lock(mutex_);
        load_state_ = LoadState::kLoading;
    }
    loader_ = std::thread(&Engine::LoadModel, this);   // 加载放到后台线程
}

void Engine::LoadModel() {
    ALOGI("loading runtime %s and model %s",
          library_path_.c_str(), model_path_.c_str());
    bool ok = LoadRuntime();
    LLMHandle handle = nullptr;
    if (ok) {
        RKLLMParam param = create_default_param_();
        // 关键参数：
        //   max_context_len = max_new_tokens = 4096（须等于模型 max_context_limit）
        //   is_async = false；n_batch = 1；embed_flash = 1
        //   enabled_cpus_mask 绑定 CPU4~CPU7
        callback_.result_callback = &RkllmResultCallback;
        int rc = init_(&handle, &param, &callback_);
        if (rc != 0) {
            ALOGE("rkllm_init failed: %d", rc);
            ok = false;
        }
    }
}

bool Engine::LoadRuntime() {
    library_ = dlopen(library_path_.c_str(), RTLD_NOW | RTLD_LOCAL);
    if (library_ == nullptr) {
        ALOGE("dlopen(%s) failed: %s", library_path_.c_str(), dlerror());
        return false;
    }
    const bool core_ok =
            ResolveSymbol(library_, "rkllm_createDefaultParam", &create_default_param_) &&
            ResolveSymbol(library_, "rkllm_init", &init_) &&
            ResolveSymbol(library_, "rkllm_run", &run_) &&
            ResolveSymbol(library_, "rkllm_abort", &abort_) &&
            ResolveSymbol(library_, "rkllm_destroy", &destroy_) &&
            ResolveSymbol(library_, "rkllm_set_function_tools", &set_function_tools_);
    // 可选：普通聊天时用来清掉工具配置
    ResolveSymbol(library_, "rkllm_set_chat_template", &set_chat_template_);
    return core_ok;
}

// 节选：工具配置的切换逻辑
if (options.tools.empty()) {
    if (tools_configured_) {
        // 本次是普通问答：先恢复纯聊天模板，
        // 否则上一轮的工具定义会污染这一轮的回答
        const int trc = set_chat_template_(handle, nullptr, nullptr, nullptr);
        if (trc == 0) {
            tools_configured_ = false;
            applied_tools_.clear();
        }
    }
} else if (options.tools != applied_tools_) {
    // 工具定义有变化才重新注册，避免重复设置
    const int trc = set_function_tools_(handle, kDefaultToolSystemPrompt,
                                        options.tools.c_str(), kToolCallTag);
    if (trc != 0) {
        ALOGE("rkllm_set_function_tools failed: %d", trc);
        *error = "rkllm_set_function_tools failed";
        return Status::kError;
    }
    applied_tools_ = options.tools;
    tools_configured_ = true;
}

Status Engine::Generate(const std::string& prompt, const std::string& role,
                        const RequestOptions& options, const TextSink& sink,
                        std::string* error)

// rkllm_run() is synchronous. Execute it on a worker thread so the
// request thread can keep watching for a disconnected HTTP client.
// rkllm_run_async() is not used because it requires param.is_async=1.
std::thread worker([&] {
    const int rc = run_(handle, &input, &infer, &context);
    if (rc != 0) {
        ALOGE("rkllm_run failed: %d", rc);
        context.failed.store(true);
    }
    context.done.store(true);
});

// 监视循环（伪代码）：client_gone 由回调里 sink 返回 false 置位
while (!done && !stopping_) {
    cv_.wait_for(lock, 100ms);
    if (client_gone && !aborted) { unlock; abort_(handle); lock; aborted = true; }
}

// C linkage is required by LLMResultCallback. The per-request context is
// passed as userdata to rkllm_run(), so no global state is involved.
extern "C" int RkllmResultCallback(RKLLMResult* result, void* userdata,
                                   LLMCallState state) {
    auto* context = static_cast<RunContext*>(userdata);
    if (context == nullptr) {
        return 0;
    }
    if (state == RKLLM_RUN_NORMAL) {
        if (result != nullptr && result->text != nullptr &&
            !context->client_gone.load()) {
            if (!(*context->sink)(std::string(result->text))) {
                context->client_gone.store(true);
            }
        }
    } else if (state == RKLLM_RUN_ERROR) {
        context->failed.store(true);
    }
    if (state == RKLLM_RUN_FINISH || state == RKLLM_RUN_ERROR) {
        context->done.store(true);
    }
    return 0;
}

<tool_call>{"name":"set_brightness","arguments":{"value":150}}</tool_call>

private void sendMessage() {
    final String text = mInput.getText().toString().trim();
    mExecutor.execute(() -> runConversation(text));
}

// 只有识别为设备操作时，第一轮请求才带工具定义
final boolean withTools = RkllmIntentRouter.isDeviceAction(mContext, text);

/** Keep descriptions short and imperative: a 3B model follows precise,
 *  unambiguous descriptions much better. */
public JSONArray buildToolsJson() {
    try {
        final JSONArray tools = new JSONArray();
        tools.put(function("open_app",
                "Open an installed app by package name",
                new String[]{"package", "string", "Package name of the app to open"}));
        tools.put(function("set_brightness",
                "Set the screen brightness",
                new String[]{"value", "integer", "Brightness level from 1 to 255"}));
        tools.put(function("set_volume",
                "Set the media volume",
                new String[]{"value", "integer", "Use the exact number the user "
                        + "asked for, do not convert it to a percentage"}));
        tools.put(function("open_settings",
                "Open the system settings app", null));
        // ... 其余：go_home / go_back / wifi_toggle / bluetooth_toggle /
        //     get_volume / get_battery / open_app_by_name
        return tools;
    } catch (JSONException e) {
        Slog.e(TAG, "failed to build tool definitions", e);
        return new JSONArray();   // 降级为不带工具调用
    }
}

private void runConversation(String text) {
    final List<JSONObject> turns = new ArrayList<>();
    final JSONArray messages = new JSONArray();
    synchronized (mHistory) {
        for (JSONObject turn : mHistory) {
            messages.put(turn);       // 会话历史一起发过去
        }
    }
    messages.put(userMessage(text));
    turns.add(userMessage(text));

    final boolean withTools = RkllmIntentRouter.isDeviceAction(mContext, text);
    for (int round = 0; round < MAX_TOOL_ROUNDS; round++) {
        // 只有第一轮带工具定义
        final StreamBuffer buffer = request(messages, withTools && round == 0);
        final String spoken = buffer.displayText().trim();
        final List<String> calls = buffer.toolCalls();
        if (calls.isEmpty() || round > 0) {
            break;   // 普通回答；或总结轮不再执行新动作
        }

        final RkllmActionExecutor.ToolCall call = mActions.parse(calls.get(0));

        boolean approved = true;
        if (mActions.needsConfirmation(call.name)) {
            // 中风险动作：UI 线程弹窗，worker 用 CountDownLatch 等待
            approved = RkllmConfirmDialog.await(mContext, mWindowManager,
                    mActions.describe(call.name, call.args));
        }
        final RkllmActionExecutor.ToolResult result = approved
                ? mActions.execute(call)
                : new RkllmActionExecutor.ToolResult(false, "declined by user");

        turns.add(assistantMessage("[device action] " + call.name + " -> "
                + result.detail));
        // 回传结果让模型做总结；原始 tool_call 文本不重放，
        // 否则后续不带工具定义的请求里模型会照着模仿
        messages.put(assistantMessage(spoken.isEmpty()
                ? "[device action] " + call.name : spoken));
        messages.put(resultMessage(call, result));
    }
    rememberTurns(turns);
}

private StreamBuffer request(JSONArray messages, boolean withTools) throws Exception {
    final JSONObject body = new JSONObject();
    body.put("model", MODEL);
    body.put("messages", messages);
    body.put("stream", true);
    body.put("max_tokens", MAX_TOKENS);
    if (withTools) {
        body.put("tools", mActions.buildToolsJson());
    }
    // ... 此处省略 HttpURLConnection 建连与写 body

    final StreamBuffer buffer = new StreamBuffer();
    try (BufferedReader reader = new BufferedReader(new InputStreamReader(
            connection.getInputStream(), StandardCharsets.UTF_8))) {
        String line;
        while ((line = reader.readLine()) != null) {
            if (!line.startsWith("data: ")) {
                continue;
            }
            final String data = line.substring(6).trim();
            if ("[DONE]".equals(data)) {
                break;
            }
            final JSONObject chunk = new JSONObject(data);
            if (chunk.has("error")) {
                appendLog("\n" + mContext.getString(R.string.rkllm_chat_error)
                        + chunk.getJSONObject("error").optString("message") + "\n");
                break;
            }
            final JSONArray choices = chunk.optJSONArray("choices");
            if (choices == null || choices.length() == 0) {
                continue;
            }
            final JSONObject delta = choices.getJSONObject(0).optJSONObject("delta");
            if (delta != null) {
                final String content = delta.optString("content", "");
                if (!content.isEmpty()) {
                    buffer.append(content, display);
                }
            }
        }
    }
    buffer.finish(display);
    return buffer;
}

private void drain(Consumer<String> display) {
    // 一个 SSE chunk 里可能同时出现：正文 + 完整调用 + 又一段正文，甚至多个调用
    while (true) {
        if (!mInTool) {
            final int start = mPending.indexOf(OPEN);
            if (start >= 0) {
                if (start > 0) {
                    show(mPending.substring(0, start), display);
                }
                mPending.delete(0, start + OPEN.length());  // 丢掉开标签
                mInTool = true;                             // 切换到工具态
                continue;
            }
            holdBackPartial(OPEN, null, display);
            return;
        }
        final int end = mPending.indexOf(CLOSE);
        if (end >= 0) {
            mToolBody.append(mPending.substring(0, end));  // 闭合标签前即调用体
            mToolCalls.add(mToolBody.toString());
            mToolBody.setLength(0);
            mPending.delete(0, end + CLOSE.length());
            mInTool = false;
            continue;
        }
        holdBackPartial(CLOSE, mToolBody, null);
        return;
    }
}

chunk1: ... 好的 <tool
chunk2: _call>{"name":"set_brightness"
chunk3: ,"arguments":{"value":150}}</tool
chunk4: _call>

// Emits everything except a possible trailing partial tag,
// so a tag split across two chunks is still recognised later.
private void holdBackPartial(String tag, StringBuilder collect,
        Consumer<String> display) {
    int hold = 0;
    final String text = mPending.toString();
    final int max = Math.min(text.length(), tag.length() - 1);
    for (int len = max; len > 0; len--) {
        if (text.regionMatches(text.length() - len, tag, 0, len)) {
            hold = len;   // 尾部 len 个字符 == 标签的前 len 个字符
            break;
        }
    }
    final int safe = mPending.length() - hold;
    if (safe <= 0) {
        return;
    }
    final String safeText = mPending.substring(0, safe);
    if (collect != null) {
        collect.append(safeText);      // 工具态：进调用体
    } else {
        show(safeText, display);      // 文本态：显示
    }
    mPending.delete(0, safe);         // 只留尾部 hold 个字符
}

[ "{\"name\":\"set_brightness\",\"arguments\":{\"value\":150}}" ]

public ToolResult execute(ToolCall call) {
    if (call == null) {
        return fail("malformed tool call");
    }
    try {
        switch (call.name) {
            case "open_app":          return openApp(call.args);
            case "set_brightness":    return setBrightness(call.args);
            case "set_volume":        return setVolume(call.args);
            case "open_settings":     return openSettings();
            case "go_home":           return goHome();
            case "go_back":           return goBack();
            case "wifi_toggle":       return wifiToggle(call.args);
            case "bluetooth_toggle":  return bluetoothToggle(call.args);
            case "get_volume":        return getVolume();
            case "get_battery":       return getBattery();
            case "open_app_by_name":  return openAppByName(call.args);
            default:                  return fail("unknown action: " + call.name);
        }
    } catch (SecurityException | IllegalArgumentException e) {
        return fail(call.name + ": " + e.getMessage());
    }
}