| import torch | |
| from openenv.core.env_server.interfaces import Message | |
| class MockTokenizer: | |
| """Mock tokenizer for testing without requiring transformers library.""" | |
| def apply_chat_template( | |
| self, | |
| conversation: list[Message], | |
| tokenize: bool = True, | |
| return_tensors: str | None = None, | |
| **kwargs, | |
| ): | |
| """Mock implementation that creates deterministic token tensors from text.""" | |
| # Concatenate all message content | |
| text = " ".join([msg["content"] for msg in conversation]) | |
| # Create deterministic tokens based on text content | |
| # Use character codes modulo 256 to get valid token IDs | |
| tokens = [ord(c) % 256 for c in text] | |
| if return_tensors == "pt": | |
| return torch.tensor([tokens]) | |
| return tokens | |
| def decode(self, token_ids, skip_special_tokens: bool = False, **kwargs) -> str: | |
| """Mock decode that reverses the encoding process.""" | |
| if isinstance(token_ids, torch.Tensor): | |
| token_ids = token_ids.tolist() | |
| # Reverse the encoding: convert tokens back to characters | |
| chars = [chr(t) for t in token_ids] | |
| return "".join(chars) | |