Error occurs when using List[Literal[1, 0]]

[mejessie]

import dspy
from typing import List, Literal

# Agent setup

# A signature
class ItemDetectionSig(dspy.Signature):
    '''Given a list of product names, evaluate each item to determine whether it represents a valid product name.
    Return a list of the same length as the input length, where each item is labeled 'Yes' if it is a valid product name and 'No' otherwise.
    For instance, if the input list contains five items, the output list should also have five items, each corresponding directly to the validity of the input item.'''
    
    items: list[str] = dspy.InputField(prefix='Items:', desc='list of comma-separated product names')
    length: int = dspy.InputField(prefix='Length:', desc='length of the list')
    responses: List[Literal[1, 0]] = dspy.OutputField()

# A module
ItemDetectionModule = dspy.Predict(ItemDetectionSig)

From the above code, I need the output in a list of yes or no.
The error occurs on line List[Literal[1, 0]] = dspy.OutputField()

Traceback (most recent call last):
  File "C:\Users\Jedsada\anaconda3\envs\ai01\Lib\site-packages\dspy\adapters\chat_adapter.py", line 78, in parse
    fields[k] = parse_value(v, signature.output_fields[k].annotation) if _parse_values else v
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Jedsada\anaconda3\envs\ai01\Lib\site-packages\dspy\adapters\chat_adapter.py", line 164, in parse_value
    return TypeAdapter(annotation).validate_python(parsed_value)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Jedsada\anaconda3\envs\ai01\Lib\site-packages\pydantic\type_adapter.py", line 412, in validate_python
    return self.validator.validate_python(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pydantic_core._pydantic_core.ValidationError: 1 validation error for list[literal[1,0]]
  Input should be a valid list [type=list_type, input_value='', input_type=str]
    For further information visit https://errors.pydantic.dev/2.10/v/list_type

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\Jedsada\anaconda3\envs\ai01\Lib\site-packages\dspy\adapters\base.py", line 25, in __call__
    value = self.parse(signature, output, _parse_values=_parse_values)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Jedsada\anaconda3\envs\ai01\Lib\site-packages\dspy\utils\callback.py", line 202, in wrapper
    return fn(instance, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Jedsada\anaconda3\envs\ai01\Lib\site-packages\dspy\adapters\chat_adapter.py", line 80, in parse
    raise ValueError(
ValueError: Error parsing field responses: 1 validation error for list[literal[1,0]]
  Input should be a valid list [type=list_type, input_value='', input_type=str]
    For further information visit https://errors.pydantic.dev/2.10/v/list_type.

                On attempting to parse the value

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\Jedsada\anaconda3\envs\ai01\main\app.py", line 55, in <module>
    responses = ItemDetectionModule(items=chunk, length=n_chunk).responses
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Jedsada\anaconda3\envs\ai01\Lib\site-packages\dspy\utils\callback.py", line 202, in wrapper
    return fn(instance, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Jedsada\anaconda3\envs\ai01\Lib\site-packages\dspy\predict\predict.py", line 154, in __call__
    return self.forward(**kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Jedsada\anaconda3\envs\ai01\Lib\site-packages\dspy\predict\predict.py", line 188, in forward
    completions = v2_5_generate(lm, config, signature, demos, kwargs, _parse_values=self._parse_values)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Jedsada\anaconda3\envs\ai01\Lib\site-packages\dspy\predict\predict.py", line 295, in v2_5_generate
    return adapter(
           ^^^^^^^^
  File "C:\Users\Jedsada\anaconda3\envs\ai01\Lib\site-packages\dspy\adapters\base.py", line 33, in __call__
    return JSONAdapter()(lm, lm_kwargs, signature, demos, inputs, _parse_values=_parse_values)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Jedsada\anaconda3\envs\ai01\Lib\site-packages\dspy\adapters\json_adapter.py", line 48, in __call__
    value = self.parse(signature, output, _parse_values=_parse_values)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Jedsada\anaconda3\envs\ai01\Lib\site-packages\dspy\utils\callback.py", line 202, in wrapper
    return fn(instance, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Jedsada\anaconda3\envs\ai01\Lib\site-packages\dspy\adapters\json_adapter.py", line 82, in parse
    fields = {k: v for k, v in fields.items() if k in signature.output_fields}
                               ^^^^^^^^^^^^
AttributeError: 'list' object has no attribute 'items'

How do I fix that ?

[okhat]

Your code works fine for me with gpt-4o-mini and this:

ItemDetectionModule(items=['apple', 'banana', 'carrot', 'dog', 'elephant'], length=5)

Can you provide an example of it breaking? (Also do make sure you're on latest DSPy via pip install -U dspy)

[amaanansari]

Version: 2.5.41
I'm getting a similar issue with the following. I'm using a float as output.

class SimpleScorerAgentSignature(dspy.Signature):
    """Predict the similarity score between the groundtruth and prediction. Focus on how similar the content is only. Give a high score if the content is the same. Give a low score if the content is different. Provide a score between 0 and 1.
    """
    groundtruth: str = dspy.InputField()
    prediction: str = dspy.InputField()
    score: float = dspy.OutputField()

Error:

Average Metric: 0.00 / 8 (0.0%):  38%|███▊      | 3/8 [00:27<00:45,  9.15s/it]2024/11/28 22:29:35 ERROR dspy.utils.parallelizer: Error processing item Example({'ConstructName': 'Calculate the range from a list of data', 'SubjectName': 'Range and Interquartile Range from a List of Data', 'CorrectAnswer': 'Only\r\nKatie', 'QuestionText': "Tom and Katie are discussing the \\( 5 \\) plants with these heights:\r\n\\( 24 \\mathrm{~cm}, 17 \\mathrm{~cm}, 42 \\mathrm{~cm}, 26 \\mathrm{~cm}, 13 \\mathrm{~cm} \\)\r\nTom says if all the plants were cut in half, the range wouldn't change.\r\nKatie says if all the plants grew by \\( 3 \\mathrm{~cm} \\) each, the range wouldn't change.\r\nWho do you agree with?", 'AnswerText': 'Both Tom and Katie', 'MisconceptionText': 'Believes if you changed all values by the same proportion the range would not change', 'MisconceptionId': 1287.0}) (input_keys={'SubjectName', 'ConstructName', 'CorrectAnswer', 'AnswerText', 'QuestionText'}): litellm.APIError: APIError: OpenAIException - Invalid response object Traceback (most recent call last):
  File "D:\no_backup\python_environments\multi-agent-misconceptions\Lib\site-packages\dspy\adapters\chat_adapter.py", line 78, in parse
    fields[k] = parse_value(v, signature.output_fields[k].annotation) if _parse_values else v
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "D:\no_backup\python_environments\multi-agent-misconceptions\Lib\site-packages\dspy\adapters\chat_adapter.py", line 164, in parse_value
    return TypeAdapter(annotation).validate_python(parsed_value)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "D:\no_backup\python_environments\multi-agent-misconceptions\Lib\site-packages\pydantic\type_adapter.py", line 412, in validate_python
    return self.validator.validate_python(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pydantic_core._pydantic_core.ValidationError: 1 validation error for float
  Input should be a valid number, unable to parse string as a number [type=float_parsing, input_value='0.7\n\n[[ ## completed', input_type=str]
    For further information visit https://errors.pydantic.dev/2.10/v/float_parsing

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "D:\no_backup\python_environments\multi-agent-misconceptions\Lib\site-packages\dspy\adapters\base.py", line 25, in __call__
    value = self.parse(signature, output, _parse_values=_parse_values)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "D:\no_backup\python_environments\multi-agent-misconceptions\Lib\site-packages\dspy\utils\callback.py", line 202, in wrapper
    return fn(instance, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "D:\no_backup\python_environments\multi-agent-misconceptions\Lib\site-packages\dspy\adapters\chat_adapter.py", line 80, in parse
    raise ValueError(
ValueError: Error parsing field score: 1 validation error for float
  Input should be a valid number, unable to parse string as a number [type=float_parsing, input_value='0.7\n\n[[ ## completed', input_type=str]
    For further information visit https://errors.pydantic.dev/2.10/v/float_parsing.

On attempting to parse the value 

0.7

[[ ## completed

[amaanansari]

Update:

If I change my code to this (not specifying float as output) it works:

class SimpleScorerAgentSignature(dspy.Signature):
    """Predict the similarity score between the groundtruth and prediction. Focus on how similar the content is only. Give a high score if the content is the same. Give a low score if the content is different. Provide a score between 0 and 1.
    """
    groundtruth: str = dspy.InputField()
    prediction: str = dspy.InputField()
    score = dspy.OutputField()

# Evaluating the answer
class EvaluationManager:
    # def __init__(self):
    #     self.misconception_db = MisconceptionDB(pathlib.Path("/data/misconception_mapping.csv"))
    #     self.misconceptions_df = DataManager.get_misconceptions(pathlib.Path("/data/misconception_mapping.csv"))

    @staticmethod
    def metric(gold, pred, trace=None):
        prediction = dspy.ChainOfThought(SimpleScorerAgentSignature)(groundtruth=gold.MisconceptionText, prediction=pred)
        # Ensure prediction_score is a float


        try:
            score = float(prediction.score.strip())
        except:
            score = 0.0
        return max(0.0, min(1.0, score))

[mejessie]

Your code works fine for me with gpt-4o-mini and this:

ItemDetectionModule(items=['apple', 'banana', 'carrot', 'dog', 'elephant'], length=5)

Can you provide an example of it breaking? (Also do make sure you're on latest DSPy via pip install -U dspy)

I used 'Ollama-llama3.1,' which worked perfectly, but when I switched to 'llama-3.1-sonar-small-128k-online,' it did not work well.