문서에서 구조화된 데이터 추출하기

python extract.py paper.pdf

pip install requests
export VENICE_API_KEY="your-api-key-here"

curl -L -o paper.pdf https://arxiv.org/pdf/1706.03762

from __future__ import annotations

import base64
import json
import os
import subprocess
import sys

import requests

BASE_URL = "https://api.venice.ai/api/v1"
AUTH = {"Authorization": f"Bearer {os.environ['VENICE_API_KEY']}"}
JSON_HEADERS = {**AUTH, "Content-Type": "application/json"}

parsed = parse_document("paper.pdf")
print(parsed["tokens"], "tokens,", len(parsed["text"]), "characters")
print(parsed["text"][:180])

12346 tokens, 39505 characters
Provided proper attribution is provided, Google hereby grants permission to
reproduce the tables and figures in this paper solely for use in journalistic or
scholarly works.
Attention Is All You Need

PAPER_SCHEMA = {
    "type": "object",
    "properties": {
        "title": {"type": "string"},
        "authors": {
            "type": "array",
            "items": {
                "type": "object",
                "properties": {
                    "name": {"type": "string"},
                    "affiliation": {"type": "string"},
                },
                "required": ["name", "affiliation"],
                "additionalProperties": False,
            },
        },
        "year": {"type": "integer"},
    },
    "required": ["title", "authors", "year"],
    "additionalProperties": False,
}

def default_model(trait: str) -> str:
    response = requests.get(f"{BASE_URL}/models/traits", headers=AUTH, timeout=30)
    response.raise_for_status()
    return response.json()["data"][trait]


def extract_from_text(text: str, schema: dict, budget: int = 12000) -> dict:
    response = requests.post(
        f"{BASE_URL}/chat/completions",
        headers=JSON_HEADERS,
        json={
            "model": default_model("default"),
            "messages": [
                {
                    "role": "system",
                    "content": (
                        "Extract the requested fields from the document. "
                        "Use only what appears in it."
                    ),
                },
                {"role": "user", "content": text[:budget]},
            ],
            "response_format": {
                "type": "json_schema",
                "json_schema": {"name": "record", "strict": True, "schema": schema},
            },
            "temperature": 0,
            "max_completion_tokens": 1500,
            "venice_parameters": {
                "include_venice_system_prompt": False,
                "disable_thinking": True,
            },
        },
        timeout=300,
    )
    response.raise_for_status()
    return read_record(response.json())

def read_record(body: dict) -> dict:
    choice = body["choices"][0]
    if choice["finish_reason"] == "length":
        raise RuntimeError(
            "Ran out of completion tokens. The JSON is truncated, not invalid. "
            "Raise max_completion_tokens or shrink the schema."
        )
    content = choice["message"].get("content")
    if not content:
        raise RuntimeError(f"Empty response, finish_reason={choice['finish_reason']}.")
    return json.loads(content)

record = extract_from_text(parsed["text"], PAPER_SCHEMA)
print(json.dumps(record, indent=2, ensure_ascii=False))

{
  "title": "Attention Is All You Need",
  "authors": [
    { "name": "Ashish Vaswani", "affiliation": "Google Brain" },
    { "name": "Noam Shazeer", "affiliation": "Google Brain" },
    { "name": "Niki Parmar", "affiliation": "Google Research" },
    { "name": "Jakob Uszkoreit", "affiliation": "Google Research" },
    { "name": "Llion Jones", "affiliation": "Google Research" },
    { "name": "Aidan N. Gomez", "affiliation": "University of Toronto" },
    { "name": "Łukasz Kaiser", "affiliation": "Google Brain" },
    { "name": "Illia Polosukhin", "affiliation": "" }
  ],
  "year": 2017
}

curl -X POST https://api.venice.ai/api/v1/augment/text-parser \
  -H "Authorization: Bearer $VENICE_API_KEY" \
  -F "file=@./scanned.pdf"

{ "error": "No text content could be extracted from the file." }

def render_first_page(pdf_path: str, png_path: str, width: int = 1400) -> None:
    """macOS only. Use pdftoppm from poppler, or pypdfium2, elsewhere."""
    subprocess.run(
        ["sips", "-s", "format", "png", "--resampleWidth", str(width),
         pdf_path, "--out", png_path],
        check=True, capture_output=True,
    )


def extract_from_image(png_path: str, schema: dict) -> dict:
    encoded = base64.b64encode(open(png_path, "rb").read()).decode()
    response = requests.post(
        f"{BASE_URL}/chat/completions",
        headers=JSON_HEADERS,
        json={
            "model": default_model("default_vision"),
            "messages": [
                {
                    "role": "system",
                    "content": (
                        "Extract the requested fields from the page image. "
                        "Use only what appears in it."
                    ),
                },
                {
                    "role": "user",
                    "content": [
                        {"type": "text", "text": "Extract the fields."},
                        {
                            "type": "image_url",
                            "image_url": {"url": f"data:image/png;base64,{encoded}"},
                        },
                    ],
                },
            ],
            "response_format": {
                "type": "json_schema",
                "json_schema": {"name": "record", "strict": True, "schema": schema},
            },
            "temperature": 0,
            "max_completion_tokens": 1500,
            "venice_parameters": {
                "include_venice_system_prompt": False,
                "disable_thinking": True,
            },
        },
        timeout=300,
    )
    response.raise_for_status()
    return read_record(response.json())

def extract(path: str, schema: dict) -> dict:
    try:
        text = parse_document(path)["text"]
    except requests.HTTPError as error:
        if error.response.status_code != 400:
            raise
        print("no extractable text, falling back to vision", file=sys.stderr)
        png = path.rsplit(".", 1)[0] + "-page1.png"
        render_first_page(path, png)
        return extract_from_image(png, schema)
    return extract_from_text(text, schema)


if __name__ == "__main__":
    document = sys.argv[1] if len(sys.argv) > 1 else "paper.pdf"
    print(json.dumps(extract(document, PAPER_SCHEMA), indent=2, ensure_ascii=False))

INVOICE_SCHEMA = {
    "type": "object",
    "properties": {
        "invoice_number": {"type": "string"},
        "issued_on": {"type": "string", "description": "ISO 8601 date"},
        "currency": {"type": "string", "description": "ISO 4217 code"},
        "total": {"type": "number"},
        "line_items": {
            "type": "array",
            "items": {
                "type": "object",
                "properties": {
                    "description": {"type": "string"},
                    "quantity": {"type": "number"},
                    "unit_price": {"type": "number"},
                },
                "required": ["description", "quantity", "unit_price"],
                "additionalProperties": False,
            },
        },
    },
    "required": ["invoice_number", "issued_on", "currency", "total", "line_items"],
    "additionalProperties": False,
}