nam-2pir

Untitled

Oct 22nd, 2025
54
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.51 KB | None | 0 0
  1. import dspy
  2. import os
  3. import httpx
  4. from twopir.llms.base import (
  5. ModelEntry,
  6. ChatMessages,
  7. LLM,
  8. )
  9.  
  10. COSTS = {
  11. "amazon.nova-lite-v1:0": {
  12. "per_input_token_cost": 0.06 / 1000000.0,
  13. "per_output_token_cost": 0.24 / 1000000.0,
  14. },
  15. }
  16.  
  17.  
  18. class BedrockLlm(LLM):
  19. """Bedrock LLM.
  20. This class requires AWS_BEARER_TOKEN_BEDROCK to be in the environment. It will call models hosted there.
  21. """
  22.  
  23. def get_id(self) -> str:
  24. return f"bedrock:{self.name}"
  25.  
  26. def __init__(self, name: str, client: httpx.AsyncClient) -> None:
  27. self.name = name
  28. self._client = client
  29. self._per_input_token_cost = COSTS[self.name]["per_input_token_cost"]
  30. self._per_output_token_cost = COSTS[self.name]["per_output_token_cost"]
  31.  
  32. def _to_converse_format(self, messages: ChatMessages) -> list:
  33. system = None
  34. converted_messages = []
  35. for message in messages:
  36. if message["role"] == "system":
  37. system = [{"text": message["content"]}]
  38. else:
  39. converted_messages.append(
  40. {
  41. "role": message["role"],
  42. "content": [{"text": message["content"]}], # type: ignore
  43. }
  44. )
  45. return system, converted_messages
  46.  
  47. async def do_generate(
  48. self,
  49. messages: ChatMessages,
  50. **kwargs,
  51. ) -> str:
  52. system, converted_messages = self._to_converse_format(messages)
  53. json_data = {
  54. "messages": converted_messages,
  55. "inferenceConfig": {
  56. "maxTokens": self.max_tokens,
  57. "temperature": self.temperature,
  58. },
  59. }
  60. if system:
  61. json_data["system"] = system
  62. response = await self._client.post(
  63. f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{self.name}/converse",
  64. json=json_data,
  65. headers={
  66. "Content-Type": "application/json",
  67. "Authorization": f"Bearer {os.environ['AWS_BEARER_TOKEN_BEDROCK']}",
  68. },
  69. )
  70. response.raise_for_status()
  71. return response.json()["output"]["message"]["content"][0]["text"]
  72.  
  73. def dspy_lm(self) -> dspy.LM:
  74. raise ValueError("Not implemented yet!")
  75.  
  76.  
  77. NOVA_LITE = ModelEntry(
  78. name="nova-lite",
  79. get_with_client=lambda client: BedrockLlm(
  80. name="amazon.nova-lite-v1:0",
  81. client=client,
  82. ),
  83. )
  84.  
Advertisement
Add Comment
Please, Sign In to add comment