mirror of
https://github.com/DS4SD/docling.git
synced 2025-12-08 20:58:11 +00:00
feat: Add support for CSV input with new backend to transform CSV files to DoclingDocument (#945)
* feat: Implement csv backend and format detection Signed-off-by: Tobias Strebitzer <tobias.strebitzer@magloft.com> * test: Implement csv parsing and format tests Signed-off-by: Tobias Strebitzer <tobias.strebitzer@magloft.com> * docs: Add example and CSV format documentation Signed-off-by: Tobias Strebitzer <tobias.strebitzer@magloft.com> * feat: Add support for various CSV dialects and update documentation Signed-off-by: Tobias Strebitzer <tobias.strebitzer@magloft.com> * feat: Add validation for delimiters and tests for inconsistent csv files Signed-off-by: Tobias Strebitzer <tobias.strebitzer@magloft.com> --------- Signed-off-by: Tobias Strebitzer <tobias.strebitzer@magloft.com>
This commit is contained in:
Tobias Strebitzer
committed by
GitHub
GitHub
parent
7493d5b01f
commit
00d9405b0a
125
docling/backend/csv_backend.py
Normal file
125
docling/backend/csv_backend.py
Normal file
@@ -0,0 +1,125 @@
|
||||
import csv
|
||||
import logging
|
||||
import warnings
|
||||
from io import BytesIO, StringIO
|
||||
from pathlib import Path
|
||||
from typing import Set, Union
|
||||
|
||||
from docling_core.types.doc import DoclingDocument, DocumentOrigin, TableCell, TableData
|
||||
|
||||
from docling.backend.abstract_backend import DeclarativeDocumentBackend
|
||||
from docling.datamodel.base_models import InputFormat
|
||||
from docling.datamodel.document import InputDocument
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CsvDocumentBackend(DeclarativeDocumentBackend):
|
||||
content: StringIO
|
||||
|
||||
def __init__(self, in_doc: "InputDocument", path_or_stream: Union[BytesIO, Path]):
|
||||
super().__init__(in_doc, path_or_stream)
|
||||
|
||||
# Load content
|
||||
try:
|
||||
if isinstance(self.path_or_stream, BytesIO):
|
||||
self.content = StringIO(self.path_or_stream.getvalue().decode("utf-8"))
|
||||
elif isinstance(self.path_or_stream, Path):
|
||||
self.content = StringIO(self.path_or_stream.read_text("utf-8"))
|
||||
self.valid = True
|
||||
except Exception as e:
|
||||
raise RuntimeError(
|
||||
f"CsvDocumentBackend could not load document with hash {self.document_hash}"
|
||||
) from e
|
||||
return
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
return self.valid
|
||||
|
||||
@classmethod
|
||||
def supports_pagination(cls) -> bool:
|
||||
return False
|
||||
|
||||
def unload(self):
|
||||
if isinstance(self.path_or_stream, BytesIO):
|
||||
self.path_or_stream.close()
|
||||
self.path_or_stream = None
|
||||
|
||||
@classmethod
|
||||
def supported_formats(cls) -> Set[InputFormat]:
|
||||
return {InputFormat.CSV}
|
||||
|
||||
def convert(self) -> DoclingDocument:
|
||||
"""
|
||||
Parses the CSV data into a structured document model.
|
||||
"""
|
||||
|
||||
# Detect CSV dialect
|
||||
head = self.content.readline()
|
||||
dialect = csv.Sniffer().sniff(head, ",;\t|:")
|
||||
_log.info(f'Parsing CSV with delimiter: "{dialect.delimiter}"')
|
||||
if not dialect.delimiter in {",", ";", "\t", "|", ":"}:
|
||||
raise RuntimeError(
|
||||
f"Cannot convert csv with unknown delimiter {dialect.delimiter}."
|
||||
)
|
||||
|
||||
# Parce CSV
|
||||
self.content.seek(0)
|
||||
result = csv.reader(self.content, dialect=dialect, strict=True)
|
||||
self.csv_data = list(result)
|
||||
_log.info(f"Detected {len(self.csv_data)} lines")
|
||||
|
||||
# Ensure uniform column length
|
||||
expected_length = len(self.csv_data[0])
|
||||
is_uniform = all(len(row) == expected_length for row in self.csv_data)
|
||||
if not is_uniform:
|
||||
warnings.warn(
|
||||
f"Inconsistent column lengths detected in CSV data. "
|
||||
f"Expected {expected_length} columns, but found rows with varying lengths. "
|
||||
f"Ensure all rows have the same number of columns."
|
||||
)
|
||||
|
||||
# Parse the CSV into a structured document model
|
||||
origin = DocumentOrigin(
|
||||
filename=self.file.name or "file.csv",
|
||||
mimetype="text/csv",
|
||||
binary_hash=self.document_hash,
|
||||
)
|
||||
|
||||
doc = DoclingDocument(name=self.file.stem or "file.csv", origin=origin)
|
||||
|
||||
if self.is_valid():
|
||||
# Convert CSV data to table
|
||||
if self.csv_data:
|
||||
num_rows = len(self.csv_data)
|
||||
num_cols = max(len(row) for row in self.csv_data)
|
||||
|
||||
table_data = TableData(
|
||||
num_rows=num_rows,
|
||||
num_cols=num_cols,
|
||||
table_cells=[],
|
||||
)
|
||||
|
||||
# Convert each cell to TableCell
|
||||
for row_idx, row in enumerate(self.csv_data):
|
||||
for col_idx, cell_value in enumerate(row):
|
||||
cell = TableCell(
|
||||
text=str(cell_value),
|
||||
row_span=1, # CSV doesn't support merged cells
|
||||
col_span=1,
|
||||
start_row_offset_idx=row_idx,
|
||||
end_row_offset_idx=row_idx + 1,
|
||||
start_col_offset_idx=col_idx,
|
||||
end_col_offset_idx=col_idx + 1,
|
||||
col_header=row_idx == 0, # First row as header
|
||||
row_header=False,
|
||||
)
|
||||
table_data.table_cells.append(cell)
|
||||
|
||||
doc.add_table(data=table_data)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Cannot convert doc with {self.document_hash} because the backend failed to init."
|
||||
)
|
||||
|
||||
return doc
|
||||
@@ -39,6 +39,7 @@ class InputFormat(str, Enum):
|
||||
PDF = "pdf"
|
||||
ASCIIDOC = "asciidoc"
|
||||
MD = "md"
|
||||
CSV = "csv"
|
||||
XLSX = "xlsx"
|
||||
XML_USPTO = "xml_uspto"
|
||||
JSON_DOCLING = "json_docling"
|
||||
@@ -61,6 +62,7 @@ FormatToExtensions: Dict[InputFormat, List[str]] = {
|
||||
InputFormat.XML_PUBMED: ["xml", "nxml"],
|
||||
InputFormat.IMAGE: ["jpg", "jpeg", "png", "tif", "tiff", "bmp"],
|
||||
InputFormat.ASCIIDOC: ["adoc", "asciidoc", "asc"],
|
||||
InputFormat.CSV: ["csv"],
|
||||
InputFormat.XLSX: ["xlsx"],
|
||||
InputFormat.XML_USPTO: ["xml", "txt"],
|
||||
InputFormat.JSON_DOCLING: ["json"],
|
||||
@@ -88,6 +90,7 @@ FormatToMimeType: Dict[InputFormat, List[str]] = {
|
||||
InputFormat.PDF: ["application/pdf"],
|
||||
InputFormat.ASCIIDOC: ["text/asciidoc"],
|
||||
InputFormat.MD: ["text/markdown", "text/x-markdown"],
|
||||
InputFormat.CSV: ["text/csv"],
|
||||
InputFormat.XLSX: [
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
],
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import csv
|
||||
import logging
|
||||
import re
|
||||
from enum import Enum
|
||||
@@ -296,6 +297,7 @@ class _DocumentConversionInput(BaseModel):
|
||||
mime = _DocumentConversionInput._mime_from_extension(ext)
|
||||
|
||||
mime = mime or _DocumentConversionInput._detect_html_xhtml(content)
|
||||
mime = mime or _DocumentConversionInput._detect_csv(content)
|
||||
mime = mime or "text/plain"
|
||||
formats = MimeTypeToFormat.get(mime, [])
|
||||
if formats:
|
||||
@@ -352,6 +354,8 @@ class _DocumentConversionInput(BaseModel):
|
||||
mime = FormatToMimeType[InputFormat.HTML][0]
|
||||
elif ext in FormatToExtensions[InputFormat.MD]:
|
||||
mime = FormatToMimeType[InputFormat.MD][0]
|
||||
elif ext in FormatToExtensions[InputFormat.CSV]:
|
||||
mime = FormatToMimeType[InputFormat.CSV][0]
|
||||
elif ext in FormatToExtensions[InputFormat.JSON_DOCLING]:
|
||||
mime = FormatToMimeType[InputFormat.JSON_DOCLING][0]
|
||||
elif ext in FormatToExtensions[InputFormat.PDF]:
|
||||
@@ -392,3 +396,32 @@ class _DocumentConversionInput(BaseModel):
|
||||
return "application/xml"
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _detect_csv(
|
||||
content: bytes,
|
||||
) -> Optional[Literal["text/csv"]]:
|
||||
"""Guess the mime type of a CSV file from its content.
|
||||
|
||||
Args:
|
||||
content: A short piece of a document from its beginning.
|
||||
|
||||
Returns:
|
||||
The mime type of a CSV file, or None if the content does
|
||||
not match any of the format.
|
||||
"""
|
||||
content_str = content.decode("ascii", errors="ignore").strip()
|
||||
|
||||
# Ensure there's at least one newline (CSV is usually multi-line)
|
||||
if "\n" not in content_str:
|
||||
return None
|
||||
|
||||
# Use csv.Sniffer to detect CSV characteristics
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(content_str)
|
||||
if dialect.delimiter in {",", ";", "\t", "|"}: # Common delimiters
|
||||
return "text/csv"
|
||||
except csv.Error:
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
@@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, model_validator, validate_call
|
||||
|
||||
from docling.backend.abstract_backend import AbstractDocumentBackend
|
||||
from docling.backend.asciidoc_backend import AsciiDocBackend
|
||||
from docling.backend.csv_backend import CsvDocumentBackend
|
||||
from docling.backend.docling_parse_v2_backend import DoclingParseV2DocumentBackend
|
||||
from docling.backend.html_backend import HTMLDocumentBackend
|
||||
from docling.backend.json.docling_json_backend import DoclingJSONBackend
|
||||
@@ -61,6 +62,11 @@ class FormatOption(BaseModel):
|
||||
return self
|
||||
|
||||
|
||||
class CsvFormatOption(FormatOption):
|
||||
pipeline_cls: Type = SimplePipeline
|
||||
backend: Type[AbstractDocumentBackend] = CsvDocumentBackend
|
||||
|
||||
|
||||
class ExcelFormatOption(FormatOption):
|
||||
pipeline_cls: Type = SimplePipeline
|
||||
backend: Type[AbstractDocumentBackend] = MsExcelDocumentBackend
|
||||
@@ -113,6 +119,9 @@ class PdfFormatOption(FormatOption):
|
||||
|
||||
def _get_default_option(format: InputFormat) -> FormatOption:
|
||||
format_to_default_options = {
|
||||
InputFormat.CSV: FormatOption(
|
||||
pipeline_cls=SimplePipeline, backend=CsvDocumentBackend
|
||||
),
|
||||
InputFormat.XLSX: FormatOption(
|
||||
pipeline_cls=SimplePipeline, backend=MsExcelDocumentBackend
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user