Skip to content

Commit

Permalink
remove frustrating / distracting logs
Browse files Browse the repository at this point in the history
  • Loading branch information
JoshuaPurtell committed Nov 20, 2024
1 parent 9b451ec commit e363ec4
Show file tree
Hide file tree
Showing 10 changed files with 306 additions and 27 deletions.
7 changes: 2 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
# sdk
Python sdk for Synth
# Synth Python SDK

python -m build
twine check dist/*
twine upload dist/*
Python SDK for Synth
3 changes: 2 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "synth-sdk"
version = "0.2.74"
version = "0.2.75"
description = ""
authors = [{name = "Synth AI", email = "[email protected]"}]
license = {text = "MIT"}
Expand All @@ -20,6 +20,7 @@ dependencies = [
"python-dotenv>=1.0.1",
"langfuse>=2.53.9",
"pytest>=8.3.3",
"pydantic-openapi-schema>=1.5.1",
]
classifiers = []

Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

setup(
name="synth-sdk",
version="0.2.74",
version="0.2.75",
packages=find_packages(),
install_requires=[
"opentelemetry-api",
Expand Down
11 changes: 6 additions & 5 deletions synth_sdk/provider_support/openai_lf.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import copy
import logging
from inspect import isclass
import types

Expand All @@ -19,7 +18,8 @@
from langfuse.utils.langfuse_singleton import LangfuseSingleton
from synth_sdk.tracing.trackers import synth_tracker_sync, synth_tracker_async
from pydantic import BaseModel
from synth_sdk.tracing.abstractions import MessageInputs, MessageOutputs
from synth_sdk.tracing.abstractions import MessageInputs
from synth_sdk.provider_support.suppress_logging import *

try:
import openai
Expand All @@ -28,7 +28,8 @@
"Please install OpenAI to use this feature: 'pip install openai'"
)

# CREDIT TO LANGFUSE FOR OPENSOURCING THE CODE THAT THIS IS BASED ON
# CREDIT TO LANGFUSE FOR OPEN-SOURCING THE CODE THAT THIS IS BASED ON
# USING WITH MIT LICENSE PERMISSION
# https://langfuse.com

try:
Expand All @@ -39,7 +40,7 @@
AzureOpenAI = None
OpenAI = None

log = logging.getLogger("langfuse")
#log = logging.getLogger("langfuse")


@dataclass
Expand Down Expand Up @@ -598,7 +599,7 @@ def _wrap(open_ai_resource: OpenAiDefinition, initialize, wrapped, args, kwargs)

return openai_response
except Exception as ex:
log.warning(ex)
#log.warning(ex)
model = kwargs.get("model", None) or None
generation.update(
end_time=_get_timestamp(),
Expand Down
30 changes: 30 additions & 0 deletions synth_sdk/provider_support/suppress_logging.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import logging

class ExcludeLangfuseMessagesFilter(logging.Filter):
def filter(self, record):
# Return False to exclude the record, True to include it
message = record.getMessage()
excluded_messages = [
"No observation found in the current context",
"No trace found in the current context",
"Adding event to partition",
]
return not any(msg in message for msg in excluded_messages)


# Configure root logger
root_logger = logging.getLogger()
root_logger.addFilter(ExcludeLangfuseMessagesFilter())
root_logger.setLevel(logging.ERROR)

# Configure langfuse logger
langfuse_logger = logging.getLogger("langfuse")
langfuse_logger.addFilter(ExcludeLangfuseMessagesFilter())
langfuse_logger.setLevel(logging.CRITICAL)
langfuse_logger.propagate = False

# Also configure the synth_sdk logger
synth_logger = logging.getLogger("synth_sdk")
synth_logger.addFilter(ExcludeLangfuseMessagesFilter())
synth_logger.setLevel(logging.ERROR)
synth_logger.propagate = False
51 changes: 51 additions & 0 deletions synth_sdk/tracing/abstractions.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,23 @@ def to_dict(self):


class TrainingQuestion(BaseModel):
"""
A training question for the system.
Attributes:
intent (str): The intended purpose or goal of the question
criteria (str): The evaluation criteria for the question
question_id (Optional[str]): Unique identifier for the question
Example:
```python
question = TrainingQuestion(
intent="Test basic addition",
criteria="Check if the system can add two numbers correctly",
question_id="math-001"
)
```
"""
intent: str
criteria: str
question_id: Optional[str] = None
Expand All @@ -144,6 +161,25 @@ def to_dict(self):


class RewardSignal(BaseModel):
"""
A reward signal indicating system performance.
Attributes:
question_id (Optional[str]): Reference to the associated training question
system_id (str): Identifier for the system being evaluated
reward (Union[float, int, bool]): Performance metric/score
annotation (Optional[str]): Additional notes about the reward
Example:
```python
signal = RewardSignal(
question_id="math-001",
system_id="calc-v1",
reward=1.0,
annotation="Correct addition performed"
)
```
"""
question_id: Optional[str] = None
system_id: str
reward: Union[float, int, bool]
Expand All @@ -159,6 +195,21 @@ def to_dict(self):


class Dataset(BaseModel):
"""
A collection of training questions and reward signals.
Attributes:
questions (List[TrainingQuestion]): List of training questions
reward_signals (List[RewardSignal]): List of associated reward signals
Example:
```python
dataset = Dataset(
questions=[TrainingQuestion(...)],
reward_signals=[RewardSignal(...)]
)
```
"""
questions: List[TrainingQuestion]
reward_signals: List[RewardSignal]

Expand Down
107 changes: 96 additions & 11 deletions synth_sdk/tracing/decorators.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
# synth_sdk/tracing/decorators.py
from typing import Callable, Optional, Set, Literal, Any, Dict, Tuple, Union, List
from functools import wraps
import threading
import time
import logging
import contextvars
from pydantic import BaseModel

from synth_sdk.tracing.abstractions import (
Event,
Expand Down Expand Up @@ -46,11 +43,56 @@ def trace_system_sync(
origin: Literal["agent", "environment"],
event_type: str,
log_result: bool = False,
manage_event: Literal["create", "end", "lazy_end", None] = None,
manage_event: Literal["create", "end", None] = None,
increment_partition: bool = False,
verbose: bool = False,
finetune_step: bool = True,
) -> Callable:
"""
Decorator for tracing synchronous method execution in an AI system.
Handles automatic input/output tracking and event management.
Args:
origin (Literal["agent", "environment"]): Source of the computation
- "agent": For AI/model operations
- "environment": For external system operations
event_type (str): Type of event being traced (e.g., "inference", "training")
log_result (bool, optional): Whether to log the function's return value.
Defaults to False.
manage_event (Literal["create", "end", None], optional):
Controls the lifecycle of the event associated with the traced function.
- "create": Start a new event at the beginning of the function execution.
- "end": Immediately conclude the current event once the function execution completes.
- None: Do not manage event lifecycle automatically. Event management must be handled manually if needed.
Defaults to None.
increment_partition (bool, optional): Whether to increment the trace partition.
Used to group related events. Defaults to False.
verbose (bool, optional): Enable detailed logging. Defaults to False.
finetune_step (bool, optional): Mark this trace as part of fine-tuning.
Defaults to True.
Returns:
Callable: The decorated function with tracing capabilities.
Raises:
ValueError: If:
- The decorated method is called without an instance (`self`).
- The instance lacks the required `system_id` attribute.
RuntimeError: If tracing initialization fails.
TypeError: If tracked values have invalid types.
Notes:
- Requires the decorated method to be an instance method with a `system_id` attribute.
- Automatically tracks method inputs and outputs.
- Manages thread-local storage for trace data.
- Ensures cleanup in the `finally` block to maintain trace integrity.
"""
def decorator(func: Callable) -> Callable:
@wraps(func)
def wrapper(*args, **kwargs):
Expand Down Expand Up @@ -216,7 +258,7 @@ def wrapper(*args, **kwargs):

# Handle event management after function execution
if (
manage_event in ["end", "lazy_end"]
manage_event == "end"
and event_type in _local.active_events
):
current_event = _local.active_events[event_type]
Expand Down Expand Up @@ -252,8 +294,53 @@ def trace_system_async(
verbose: bool = False,
finetune_step: bool = True,
) -> Callable:
"""Decorator for tracing asynchronous functions."""

"""
Decorator for tracing asynchronous method execution in an AI system.
Handles automatic input/output tracking and event management in async contexts.
Args:
origin (Literal["agent", "environment"]): Source of the computation
- "agent": For AI/model operations
- "environment": For external system operations
event_type (str): Type of event being traced (e.g., "inference", "training")
log_result (bool, optional): Whether to log the function's return value.
Defaults to False.
manage_event (Literal["create", "end", "lazy_end", None], optional):
Controls the lifecycle of the event associated with the traced function.
- "create": Start a new event at the beginning of the function execution.
- "end": Immediately conclude the current event once the function execution completes.
- "lazy_end": Mark the event to be concluded after all nested computations and asynchronous tasks have finished. This ensures that all related asynchronous operations are captured within the same event scope.
- None: Do not manage event lifecycle automatically. Event management must be handled manually if needed.
Defaults to None.
increment_partition (bool, optional): Whether to increment the trace partition.
Used to group related events. Defaults to False.
verbose (bool, optional): Enable detailed logging. Defaults to False.
finetune_step (bool, optional): Mark this trace as part of fine-tuning.
Defaults to True.
Returns:
Callable: The decorated asynchronous function with tracing capabilities.
Raises:
ValueError: If:
- The decorated method is called without an instance (`self`).
- The instance lacks the required `system_id` attribute.
RuntimeError: If tracing initialization fails.
TypeError: If tracked values have invalid types.
Notes:
- Requires the decorated method to be an instance method with a `system_id` attribute.
- Automatically tracks method inputs and outputs.
- Uses `contextvars` for async-safe storage.
- Ensures cleanup in the `finally` block to maintain trace integrity.
- Safe for concurrent execution in asynchronous contexts.
"""
def decorator(func: Callable) -> Callable:
@wraps(func)
async def async_wrapper(*args, **kwargs):
Expand Down Expand Up @@ -420,7 +507,7 @@ async def async_wrapper(*args, **kwargs):

# Handle event management after function execution
if (
manage_event in ["end", "lazy_end"]
manage_event == "end"
and event_type in active_events_var.get()
):
current_event = active_events_var.get()[event_type]
Expand All @@ -443,10 +530,8 @@ async def async_wrapper(*args, **kwargs):
synth_tracker_async.finalize()
# Reset context variable for system_id
system_id_var.reset(system_id_token)
logger.debug(f"Cleaning up system_id from context vars")

logger.debug("Cleaning up system_id from context vars")
return async_wrapper

return decorator

def trace_system(
Expand Down
2 changes: 1 addition & 1 deletion synth_sdk/tracing/events/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ def add_event(self, system_id: str, event: Event):
self.logger.debug(
f"Event details: opened={event.opened}, closed={event.closed}, partition={event.partition_index}"
)
print("Adding event to partition")
#print("Adding event to partition")

#try:
if not self._lock.acquire(timeout=5):
Expand Down
Loading

0 comments on commit e363ec4

Please sign in to comment.