from http import HTTPStatus
import dashscope
import os

def tokenizer():
    response = dashscope.Tokenization.call(
        model='qwen-turbo',
        messages=[{'role': 'user', 'content': '你好？'}],
        api_key=os.getenv("DASHSCOPE_API_KEY"),
        )
    if response.status_code == HTTPStatus.OK:
        print('Result is: %s' % response)
    else:
        print('Failed request_id: %s, status_code: %s, code: %s, message:%s' %
              (response.request_id, response.status_code, response.code,
               response.message))


if __name__ == '__main__':
    tokenizer()

// Copyright (c) Alibaba, Inc. and its affiliates.
import java.lang.System;
import com.alibaba.dashscope.common.Message;
import java.util.Arrays;
import com.alibaba.dashscope.aigc.generation.GenerationParam;
import com.alibaba.dashscope.common.Role;
import com.alibaba.dashscope.exception.ApiException;
import com.alibaba.dashscope.exception.InputRequiredException;
import com.alibaba.dashscope.exception.NoApiKeyException;
import com.alibaba.dashscope.tokenizers.Tokenization;
import com.alibaba.dashscope.tokenizers.TokenizationResult;
import com.alibaba.dashscope.utils.JsonUtils;

public class Main {
    public static void tokenizer() throws ApiException, NoApiKeyException, InputRequiredException {
        Tokenization tokenizer = new Tokenization();
        Message userMsg = Message.builder()
                .role(Role.USER.getValue())
                .content("你好？")
                .build();
        GenerationParam param = GenerationParam.builder()
                .model(Tokenization.Models.QWEN_TURBO)
                .messages(Arrays.asList(userMsg))
                .apiKey(System.getenv("DASHSCOPE_API_KEY")) // 如果您沒有配置環境變量，可以在這里配置API-KEY
                .build();
        TokenizationResult result = tokenizer.call(param);
        System.out.println(JsonUtils.toJson(result));
    }

    public static void main(string[] args) {
        try {
            tokenizer();
        } catch (ApiException | NoApiKeyException | InputRequiredException e) {
            System.out.println(string.format("Exception %s", e.getMessage()));
        }
        System.exit(0);
    }
}

curl --location 'https://dashscope.aliyuncs.com/api/v1/tokenizer' \
--header "Authorization: Bearer $DASHSCOPE_API_KEY" \
--header 'Content-Type: application/json' \
--data '{
    "model": "qwen-plus",
    "input":{
        "messages":[
            {
                "role": "user",
                "content": "北京有哪些好玩地方？"
            },
            {
                "role": "assistant",
                "content": "故宮、頤和園、天壇等都是可以去游玩的景點哦。"
            },
            {
                "role": "user",
                "content": "幫我安排一些行程"
            }
        ]
    },
    "parameters": {
    }
}'

{
  "output": {
    "token_ids": [
      68990,
      104719,
      108257,
      100371,
      11319,
      113508,
      5373,
      113085,
      33108,
      99354,
      5373,
      35727,
      101152,
      49567,
      100132,
      73670,
      85336,
      109280,
      9370,
      105869,
      104170,
      1773,
      108965,
      103956,
      101883,
      106318
    ],
    "tokens": [
      "北京",
      "有哪些",
      "好玩",
      "地方",
      "？",
      "故宮",
      "、",
      "頤",
      "和",
      "園",
      "、",
      "天",
      "壇",
      "等",
      "都是",
      "可以",
      "去",
      "游玩",
      "的",
      "景點",
      "哦",
      "。",
      "幫我",
      "安排",
      "一些",
      "行程"
    ]
  },
  "usage": {
    "input_tokens": 26
  },
  "request_id": "aeabe685-5e87-9c32-93f4-21396c5656f2"
}