2 - Install Dependencies¶
In [10]:
import os
if os.getenv("COLAB_RELEASE_TAG"):
print("Running on Colab")
RUNNING_ON_COLAB = True
else:
print("NOT running on Colab")
RUNNING_ON_COLAB = False
NOT running on Colab
In [11]:
## uncomment one of the following lines to install dependencies
# !pip install -q openai python-dotenv pydantic
!uv pip install -q openai python-dotenv pydantic
3 - Load Configuration¶
In [12]:
import os
if RUNNING_ON_COLAB:
from google.colab import userdata
NEBIUS_API_KEY = userdata.get('NEBIUS_API_KEY')
else:
from dotenv import load_dotenv
load_dotenv()
NEBIUS_API_KEY = os.getenv('NEBIUS_API_KEY')
if NEBIUS_API_KEY:
print('\u2705 NEBIUS_API_KEY found')
os.environ['NEBIUS_API_KEY'] = NEBIUS_API_KEY
else:
raise RuntimeError('\u274c NEBIUS_API_KEY NOT found')
✅ NEBIUS_API_KEY found
In [13]:
from pydantic import BaseModel, Field
from typing import Optional
import json
class PersonInfo(BaseModel):
name: str = Field(description="The person's full name")
age: Optional[int] = Field(description="The person's age if mentioned")
occupation: str = Field(description="The person's job or occupation")
schema = PersonInfo.model_json_schema()
print(json.dumps(schema, indent=2))
{
"properties": {
"name": {
"description": "The person's full name",
"title": "Name",
"type": "string"
},
"age": {
"anyOf": [
{
"type": "integer"
},
{
"type": "null"
}
],
"description": "The person's age if mentioned",
"title": "Age"
},
"occupation": {
"description": "The person's job or occupation",
"title": "Occupation",
"type": "string"
}
},
"required": [
"name",
"age",
"occupation"
],
"title": "PersonInfo",
"type": "object"
}
5 - Initialize API¶
We need a model that supports function calling.
- Go to models tab in tokenfactory.nebius.com
- Select text to text models
- Copy the model name
In [14]:
import os
from openai import OpenAI
MODEL = "openai/gpt-oss-20b"
client = OpenAI(
base_url="https://api.tokenfactory.nebius.com/v1",
api_key=os.getenv('NEBIUS_API_KEY')
)
6 - API Call¶
Note the following
we are passing
schemain system promptand specifying
guided_jsonon api call
extra_body={
"guided_json": schema,
},
- Also asking for JSON output
response_format={"type": "json_object"}, # Forces JSON output
And note how completion.choices[0].message.content is in JSON format
In [ ]:
system_prompt = f"""
Act as an experienced data extractor. \
Your goal is to extract the required information (is given below) from the text. \
Output the extracted information in the json format, following the required schema.
Required information schema:
```
{schema}
```
""".strip()
text = """
The task of json generation was assigned Mikhail Kashirskii, a a 34-year-old software developer at Nebius. \
His role in this task is to implement the support of json outputs of the LLM.
""".strip()
completion = client.chat.completions.create(
model=MODEL,
messages=[
{
"role": "system",
"content": system_prompt
},
{
"role": "user",
"content": text
}
],
extra_body={
"guided_json": schema,
},
response_format={"type": "json_object"}, # Forces JSON output
temperature=0.1,
#max_tokens=100,
top_p=0.9
)
print(completion.to_json())
# And note how `completion.choices[0].message.content` is in JSON format
{
"id": "chatcmpl-b20676a0fba1bcb3",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"logprobs": null,
"message": {
"content": "{\"name\":\"Mikhail Kashirskii\",\"age\":34,\"occupation\":\"software developer\"}",
"refusal": null,
"role": "assistant",
"annotations": null,
"audio": null,
"function_call": null,
"tool_calls": [],
"reasoning": "We need to extract name, age, occupation. The text: \"The task of json generation was assigned Mikhail Kashirskii, a a 34-year-old software developer at Nebius. His role in this task is to implement the support of json outputs of the LLM.\"\n\nName: Mikhail Kashirskii. Age: 34. Occupation: software developer. Output JSON with required fields. Ensure age integer.",
"reasoning_content": "We need to extract name, age, occupation. The text: \"The task of json generation was assigned Mikhail Kashirskii, a a 34-year-old software developer at Nebius. His role in this task is to implement the support of json outputs of the LLM.\"\n\nName: Mikhail Kashirskii. Age: 34. Occupation: software developer. Output JSON with required fields. Ensure age integer."
},
"stop_reason": null,
"token_ids": null
}
],
"created": 1773208203,
"model": "openai/gpt-oss-20b",
"object": "chat.completion",
"service_tier": null,
"system_fingerprint": null,
"usage": {
"completion_tokens": 115,
"prompt_tokens": 272,
"total_tokens": 387,
"completion_tokens_details": null,
"prompt_tokens_details": null
},
"prompt_logprobs": null,
"prompt_token_ids": null,
"kv_transfer_params": null
}
7 - Convert the output to PersonInfo object¶
In [16]:
raw = completion.choices[0].message.content
person = PersonInfo(**json.loads(raw))
print(person.model_dump_json(indent=2))
{
"name": "Mikhail Kashirskii",
"age": 34,
"occupation": "software developer"
}
8 - Age Optional¶
In the resulting PersonInfo, age attribute will be null.
In [17]:
text = """
The task of json generation was assigned Mikhail Kashirskii, a software developer at Nebius. \
His role in this task is to implement the support of json outputs of the LLM.
""".strip()
completion = client.chat.completions.create(
model=MODEL,
messages=[
{
"role": "system",
"content": system_prompt
},
{
"role": "user",
"content": text
}
],
extra_body={
"guided_json": schema,
},
response_format={"type": "json_object"}, # Forces JSON output
temperature=0.1,
#max_tokens=100,
top_p=0.9
)
print(completion.to_json())
{
"id": "chatcmpl-930bc56c5ee3f580",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"logprobs": null,
"message": {
"content": "{\"name\":\"Mikhail Kashirskii\",\"age\":null,\"occupation\":\"software developer at Nebius\"}",
"refusal": null,
"role": "assistant",
"annotations": null,
"audio": null,
"function_call": null,
"tool_calls": [],
"reasoning": "We need to extract name, age, occupation. The text: \"The task of json generation was assigned Mikhail Kashirskii, a software developer at Nebius. His role in this task is to implement the support of json outputs of the LLM.\"\n\nName: Mikhail Kashirskii. Occupation: software developer at Nebius. Age: not mentioned. So age null. Output JSON with name, age null, occupation string. Ensure schema: name string, age integer or null, occupation string. Provide JSON.",
"reasoning_content": "We need to extract name, age, occupation. The text: \"The task of json generation was assigned Mikhail Kashirskii, a software developer at Nebius. His role in this task is to implement the support of json outputs of the LLM.\"\n\nName: Mikhail Kashirskii. Occupation: software developer at Nebius. Age: not mentioned. So age null. Output JSON with name, age null, occupation string. Ensure schema: name string, age integer or null, occupation string. Provide JSON."
},
"stop_reason": null,
"token_ids": null
}
],
"created": 1773208204,
"model": "openai/gpt-oss-20b",
"object": "chat.completion",
"service_tier": null,
"system_fingerprint": null,
"usage": {
"completion_tokens": 137,
"prompt_tokens": 266,
"total_tokens": 403,
"completion_tokens_details": null,
"prompt_tokens_details": null
},
"prompt_logprobs": null,
"prompt_token_ids": null,
"kv_transfer_params": null
}
In [18]:
raw = completion.choices[0].message.content
person = PersonInfo(**json.loads(raw))
print(person.model_dump_json(indent=2))
{
"name": "Mikhail Kashirskii",
"age": null,
"occupation": "software developer at Nebius"
}