Merge pull request #6 from synth-laboratories/github_actions

GitHub actions

.github/workflows/github-actions-demo.yml

@@ -0,0 +1,56 @@
+name: GitHub Actions Demo
+run-name: ${{ github.actor }} is testing out GitHub Actions 🚀
+on: [push]
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Pythonc
+        uses: actions/setup-python@v4
+        with: 
+          python-version: '3.9'
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip 
+          pip install -r requirements.txt
+
+      - name: Install Local Version of synth_sdk
+        run: |
+          pip install -e .
+        
+      - name: Find Tests
+        id: find_tests
+        run: |
+          # Find all files in the 'testing' directory and join them into a space-separated string
+          # NOTE: Only run pytests on files that end with '_test.py'
+          TEST_SCRIPTS=$(find testing -type f -name '*_test.py' -print | tr '\n' ' ')
+          echo "Found test scripts: $TEST_SCRIPTS"
+          echo "TEST_SCRIPTS=$TEST_SCRIPTS" >> $GITHUB_ENV
+        env:
+          SYNTH_API_KEY: ${{ secrets.SYNTH_API_KEY }}
+          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
+          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          DUMMY_SECRET: ${{ secrets.DUMMY_SECRET }}
+      - name: Run Tests
+        run: |
+          # Execute test scripts listed in the TEST_SCRIPTS variable
+          echo "Running tests: $TEST_SCRIPTS"
+          for script in $TEST_SCRIPTS; do
+            echo "Running test $script"
+            pytest --log-level=WARNING $script
+          done
+        env:
+          SYNTH_API_KEY: ${{ secrets.SYNTH_API_KEY }}
+          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
+          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          DUMMY_SECRET: ${{ secrets.DUMMY_SECRET }}
+
+
+
+

.gitignore

@@ -160,3 +160,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# uv.lock is annoying
+uv.lock

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "synth-sdk"
-version = "0.2.60"
+version = "0.2.61"
 description = ""
 authors = [{name = "Synth AI", email = "[email protected]"}]
 license = {text = "MIT"}

requirements.txt

@@ -0,0 +1,30 @@
+#apropos-ai==0.4.5
+fastapi>=0.115.0
+opentelemetry-api>=1.27.0
+opentelemetry-instrumentation>=0.48b0
+opentelemetry-sdk>=1.27.0
+pydantic>=2.9.2
+asyncpg>=0.24.0
+
+supabase>=2.8.1
+uvicorn>=0.15.0
+redis>=4.0.0
+python-dotenv>=0.19.0
+python-jose[cryptography]>=3.3.0
+passlib>=1.7.4
+python-multipart>=0.0.5
+pytest==8.3.3
+pytest-asyncio==0.24.0
+pytest-env==1.1.5
+
+
+prefect>=3.0.4
+psycopg2-binary==2.9.6
+pydantic-settings>=2.5.2
+argon2-cffi>=23.1.0
+modal>=0.64.159
+loguru>=0.7.2
+alembic>=1.13.3
+zyk>=0.2.10
+#synth_sdk>=0.2.61
+#smallbench>=0.2.11

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name="synth-sdk",
-    version="0.2.60",
+    version="0.2.61",
     packages=find_packages(),
     install_requires=[
         "opentelemetry-api",

synth_sdk/tracing/upload.py

@@ -26,6 +26,14 @@ def validate_json(data: dict) -> None:
     except (TypeError, OverflowError) as e:
         raise ValueError(f"Contains non-JSON-serializable values: {e}. {data}")
 
+def createPayload(dataset: Dataset, traces: str) -> Dict[str, Any]:
+    payload = {
+        "traces": [
+            trace.to_dict() for trace in traces
+        ],  # Convert SystemTrace objects to dicts
+        "dataset": dataset.to_dict(),
+    }
+    return payload
 
 def send_system_traces(
     dataset: Dataset, base_url: str, api_key: str
@@ -43,12 +51,7 @@ def send_system_traces(
     # Send the traces with the token
     api_url = f"{base_url}/upload/"
 
-    payload = {
-        "traces": [
-            trace.to_dict() for trace in traces
-        ],  # Convert SystemTrace objects to dicts
-        "dataset": dataset.to_dict(),
-    }
+    payload = createPayload(dataset, traces) # Create the payload
 
     validate_json(payload)  # Validate the entire payload
 

testing/pytest_test.py

@@ -0,0 +1,13 @@
+import pytest
+
+def test_example():
+  assert 1 + 1 == 2
+
+def test_string():
+  assert "hello".upper() == "HELLO"
+
+try:
+  def test_list():
+    assert len([1, 2, 3]) == 3
+except Exception as e:
+  print(f"Error: {str(e)}")

testing/upload_sync_test.py

@@ -0,0 +1,148 @@
+from zyk import LM
+from synth_sdk.tracing.decorators import trace_system_sync, _local
+from synth_sdk.tracing.trackers import SynthTrackerSync
+from synth_sdk.tracing.upload import upload
+from synth_sdk.tracing.upload import validate_json
+from synth_sdk.tracing.upload import createPayload
+from synth_sdk.tracing.abstractions import TrainingQuestion, RewardSignal, Dataset
+from synth_sdk.tracing.events.store import event_store
+from typing import Dict
+#import asyncio
+#import synth_sdk.config.settings
+import time
+#import json
+import logging
+import pytest
+from unittest.mock import MagicMock, Mock, patch
+import requests
+
+# Configure logging
+logging.basicConfig(
+    level=logging.DEBUG,  # Changed from CRITICAL to DEBUG
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+# Unit Test Configuration: 
+# ===============================  
+questions = ["What's the capital of France?"]
+mock_llm_response = "The capital of France is Paris."
+
+# This function generates a payload from the data in the dataset to compare the sent payload against
+def generate_payload_from_data(dataset: Dataset) -> Dict:
+    traces = event_store.get_system_traces()
+
+    payload = {
+        "traces": [
+            trace.to_dict() for trace in traces
+        ],  # Convert SystemTrace objects to dicts
+        "dataset": dataset.to_dict(),
+    }
+    return payload
+
+def createPayload_wrapper(dataset: Dataset, base_url: str, api_key: str) -> Dict:
+    payload = createPayload(dataset, event_store.get_system_traces())
+
+    response = requests.Response()
+    response.status_code = 200
+
+    return response, payload
+
+# ===============================
+
+class TestAgent:
+    def __init__(self):
+        self.system_id = "test_agent_upload"
+        logger.debug("Initializing TestAgent with system_id: %s", self.system_id)
+        #self.lm = LM(model_name="gpt-4o-mini-2024-07-18", formatting_model_name="gpt-4o-mini-2024-07-18", temperature=1,)
+        self.lm = MagicMock()
+        self.lm.respond_sync.return_value = mock_llm_response
+        logger.debug("LM initialized")
+
+    @trace_system_sync(
+        origin="agent",
+        event_type="lm_call",
+        manage_event="create",
+        increment_partition=True,
+        verbose=False,
+    )
+    def make_lm_call(self, user_message: str) -> str: # Calls an LLM to respond to a user message
+        # Only pass the user message, not self
+        SynthTrackerSync.track_input([user_message], variable_name="user_message", origin="agent")
+
+        logger.debug("Starting LM call with message: %s", user_message)
+        response = self.lm.respond_sync(
+            system_message="You are a helpful assistant.", user_message=user_message
+        )
+        SynthTrackerSync.track_output(response, variable_name="response", origin="agent")
+
+        logger.debug("LM response received: %s", response)
+        #time.sleep(0.1)
+        return response
+
+    @trace_system_sync(
+        origin="environment",
+        event_type="environment_processing",
+        manage_event="create",
+        verbose=False,
+    )
+    def process_environment(self, input_data: str) -> dict:
+        # Only pass the input data, not self
+        SynthTrackerSync.track_input([input_data], variable_name="input_data", origin="environment")
+        result = {"processed": input_data, "timestamp": time.time()}
+        SynthTrackerSync.track_output(result, variable_name="result", origin="environment")
+        return result
+
+@pytest.mark.asyncio(loop_scope="session")
+@patch("synth_sdk.tracing.upload.send_system_traces", side_effect=createPayload_wrapper)
+async def test_upload(mock_send_system_traces):
+    logger.info("Starting run_test")
+    agent = TestAgent() # Create test agent
+
+    logger.debug("Test questions initialized: %s", questions) # List of test questions
+
+    # Make multiple LM calls with environment processing
+    responses = []
+    for i, question in enumerate(questions):
+        logger.info("Processing question %d: %s", i, question)
+        env_result = agent.process_environment(question)
+        logger.debug("Environment processing result: %s", env_result)
+
+        response = agent.make_lm_call(question)
+        responses.append(response)
+        logger.debug("Response received and stored: %s", response)
+
+    logger.info("Creating dataset for upload")
+    # Create dataset for upload
+    dataset = Dataset(
+        questions=[
+            TrainingQuestion(
+                intent="Test question",
+                criteria="Testing tracing functionality",
+                question_id=f"q{i}",
+            )
+            for i in range(len(questions))
+        ],
+        reward_signals=[
+            RewardSignal(
+                question_id=f"q{i}",
+                system_id=agent.system_id,
+                reward=1.0,
+                annotation="Test reward",
+            )
+            for i in range(len(questions))
+        ],
+    )
+    logger.debug(
+        "Dataset created with %d questions and %d reward signals",
+        len(dataset.questions),
+        len(dataset.reward_signals),
+    )
+
+    # Upload traces
+    logger.info("Attempting to upload traces")
+    response, payload = await upload(dataset=dataset, verbose=True)
+    logger.info("Upload successful!")
+
+    # Pytest assertion
+    assert payload == generate_payload_from_data(dataset)