Remove deprecated demo, ingest, schema, and test parser files; add new LLM parser implementation and settings configuration
This commit is contained in:
@@ -0,0 +1,28 @@
|
||||
import asyncio
|
||||
import csv
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class RowSchema(BaseModel):
|
||||
section: str
|
||||
explanation: str
|
||||
|
||||
client = AsyncOpenAI()
|
||||
|
||||
async def process_row(row):
|
||||
resp = await client.chat.completions.create(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": f"Extract relevant section:\n{row}"}],
|
||||
response_format={"type": "json_object"} # ensures JSON output
|
||||
)
|
||||
return RowSchema.model_validate_json(resp.choices[0].message.content)
|
||||
|
||||
async def main():
|
||||
with open("data.csv") as f:
|
||||
reader = csv.DictReader(f)
|
||||
tasks = [process_row(row) for row in reader]
|
||||
return await asyncio.gather(*tasks)
|
||||
|
||||
results = asyncio.run(main())
|
||||
Reference in New Issue
Block a user