mirrors/docling
1
0
Fork 0
mirror of https://github.com/DS4SD/docling.git synced 2025-12-08 12:48:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(docx): slow table parsing (#2553)

Browse Source 
* chore(docx): remove unnecessary import

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* fix(docx): simplify parsing of simple tables

Simplify the parsing of tables with just text (no rich cells).
Move nested function group_cell_elements out of _handle_tables for readability.

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* chore(docx): reuse method for finding inline pictures

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* chore(docx): format strikethrough text

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* tests(docx): use fixtures to avoid converting same file multiple times

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* fix(docx): remove unnecessary argument docx_obj in functions

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* tests(docx): add test for rich table cells

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* chore(docx): small improvements in backend and its unit tests

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

* chore(docx): parse superscript and subscript formatted text

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>

---------

Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
This commit is contained in:
Cesar Berrospi Ramis
2025-11-06 05:25:53 +01:00
committed by GitHub
parent 0ba8d5d9e3
commit ef623ffcee
6 changed files with 3366 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

262
tests/test_backend_msword.py
View File

@@ -1,3 +1,4 @@
import logging
import os
from pathlib import Path
@@ -18,23 +19,109 @@ from docling.document_converter import DocumentConverter
from .test_data_gen_flag import GEN_TEST_DATA
from .verify_utils import verify_document, verify_export
_log = logging.getLogger(__name__)
GENERATE = GEN_TEST_DATA
IS_CI = bool(os.getenv("CI"))
@pytest.fixture(scope="module")
def docx_paths() -> list[Path]:
# Define the directory you want to search
directory = Path("./tests/data/docx/")
# List all docx files in the directory and its subdirectories
docx_files = sorted(directory.rglob("*.docx"))
return docx_files
def get_converter():
converter = DocumentConverter(allowed_formats=[InputFormat.DOCX])
return converter
@pytest.fixture(scope="module")
def documents(docx_paths) -> list[tuple[Path, DoclingDocument]]:
documents: list[dict[Path, DoclingDocument]] = []
converter = get_converter()
for docx_path in docx_paths:
_log.debug(f"converting {docx_path}")
gt_path = (
docx_path.parent.parent / "groundtruth" / "docling_v2" / docx_path.name
)
conv_result: ConversionResult = converter.convert(docx_path)
doc: DoclingDocument = conv_result.document
assert doc, f"Failed to convert document from file {gt_path}"
documents.append((gt_path, doc))
return documents
def _test_e2e_docx_conversions_impl(docx_paths: list[tuple[Path, DoclingDocument]]):
has_libreoffice = False
try:
cmd = get_libreoffice_cmd(raise_if_unavailable=True)
if cmd is not None:
has_libreoffice = True
except Exception:
pass
for docx_path, doc in docx_paths:
if not IS_CI and not has_libreoffice and docx_path.name == "drawingml.docx":
print(f"Skipping {docx_path} because no Libreoffice is installed.")
continue
pred_md: str = doc.export_to_markdown()
assert verify_export(pred_md, str(docx_path) + ".md", generate=GENERATE), (
f"export to markdown failed on {docx_path}"
)
pred_itxt: str = doc._export_to_indented_text(
max_text_len=70, explicit_tables=False
)
assert verify_export(pred_itxt, str(docx_path) + ".itxt", generate=GENERATE), (
f"export to indented-text failed on {docx_path}"
)
assert verify_document(doc, str(docx_path) + ".json", generate=GENERATE), (
f"DoclingDocument verification failed on {docx_path}"
)
if docx_path.name == "word_tables.docx":
pred_html: str = doc.export_to_html()
assert verify_export(
pred_text=pred_html,
gtfile=str(docx_path) + ".html",
generate=GENERATE,
), f"export to html failed on {docx_path}"
flaky_file = "textbox.docx"
def test_e2e_docx_conversions(documents):
target = [item for item in documents if item[0].name != flaky_file]
_test_e2e_docx_conversions_impl(target)
@pytest.mark.xfail(strict=False)
def test_textbox_extraction():
in_path = Path("tests/data/docx/textbox.docx")
in_doc = InputDocument(
path_or_stream=in_path,
format=InputFormat.DOCX,
backend=MsWordDocumentBackend,
)
backend = MsWordDocumentBackend(
in_doc=in_doc,
path_or_stream=in_path,
)
doc = backend.convert()
def test_textbox_conversion(documents):
target = [item for item in documents if item[0].name == flaky_file]
_test_e2e_docx_conversions_impl(target)
@pytest.mark.xfail(strict=False)
def test_textbox_extraction(documents):
name = "textbox.docx"
doc = next(item[1] for item in documents if item[0].name == name)
# Verify if a particular textbox content is extracted
textbox_found = False
@@ -44,18 +131,9 @@ def test_textbox_extraction():
assert textbox_found
def test_heading_levels():
in_path = Path("tests/data/docx/word_sample.docx")
in_doc = InputDocument(
path_or_stream=in_path,
format=InputFormat.DOCX,
backend=MsWordDocumentBackend,
)
backend = MsWordDocumentBackend(
in_doc=in_doc,
path_or_stream=in_path,
)
doc = backend.convert()
def test_heading_levels(documents):
name = "word_sample.docx"
doc = next(item[1] for item in documents if item[0].name == name)
found_lvl_1 = found_lvl_2 = False
for item, _ in doc.iterate_items():
@@ -69,104 +147,11 @@ def test_heading_levels():
assert found_lvl_1 and found_lvl_2
def get_docx_paths():
# Define the directory you want to search
directory = Path("./tests/data/docx/")
def test_text_after_image_anchors(documents):
"""Test to analyse whether text gets parsed after image anchors."""
# List all PDF files in the directory and its subdirectories
pdf_files = sorted(directory.rglob("*.docx"))
return pdf_files
def get_converter():
converter = DocumentConverter(allowed_formats=[InputFormat.DOCX])
return converter
def _test_e2e_docx_conversions_impl(docx_paths: list[Path]):
converter = get_converter()
has_libreoffice = False
try:
cmd = get_libreoffice_cmd(raise_if_unavailable=True)
if cmd is not None:
has_libreoffice = True
except Exception:
pass
for docx_path in docx_paths:
if (
not IS_CI
and not has_libreoffice
and str(docx_path) in ("tests/data/docx/drawingml.docx",)
):
print(f"Skipping {docx_path} because no Libreoffice is installed.")
continue
gt_path = (
docx_path.parent.parent / "groundtruth" / "docling_v2" / docx_path.name
)
conv_result: ConversionResult = converter.convert(docx_path)
doc: DoclingDocument = conv_result.document
pred_md: str = doc.export_to_markdown()
assert verify_export(pred_md, str(gt_path) + ".md", generate=GENERATE), (
f"export to markdown failed on {docx_path}"
)
pred_itxt: str = doc._export_to_indented_text(
max_text_len=70, explicit_tables=False
)
assert verify_export(pred_itxt, str(gt_path) + ".itxt", generate=GENERATE), (
f"export to indented-text failed on {docx_path}"
)
assert verify_document(doc, str(gt_path) + ".json", generate=GENERATE), (
f"DoclingDocument verification failed on {docx_path}"
)
if docx_path.name == "word_tables.docx":
pred_html: str = doc.export_to_html()
assert verify_export(
pred_text=pred_html,
gtfile=str(gt_path) + ".html",
generate=GENERATE,
), f"export to html failed on {docx_path}"
flaky_path = Path("tests/data/docx/textbox.docx")
def test_e2e_docx_conversions():
_test_e2e_docx_conversions_impl(
docx_paths=[path for path in get_docx_paths() if path != flaky_path]
)
@pytest.mark.xfail(strict=False)
def test_textbox_conversion():
_test_e2e_docx_conversions_impl(docx_paths=[flaky_path])
def test_text_after_image_anchors():
"""
Test to analyse whether text gets parsed after image anchors.
"""
in_path = Path("tests/data/docx/word_image_anchors.docx")
in_doc = InputDocument(
path_or_stream=in_path,
format=InputFormat.DOCX,
backend=MsWordDocumentBackend,
)
backend = MsWordDocumentBackend(
in_doc=in_doc,
path_or_stream=in_path,
)
doc = backend.convert()
name = "word_image_anchors.docx"
doc = next(item[1] for item in documents if item[0].name == name)
found_text_after_anchor_1 = found_text_after_anchor_2 = (
found_text_after_anchor_3
@@ -188,3 +173,38 @@ def test_text_after_image_anchors():
and found_text_after_anchor_3
and found_text_after_anchor_4
)
def test_is_rich_table_cell(docx_paths):
"""Test the function is_rich_table_cell."""
name = "docx_rich_cells.docx"
path = next(item for item in docx_paths if item.name == name)
in_doc = InputDocument(
path_or_stream=path,
format=InputFormat.DOCX,
backend=MsWordDocumentBackend,
filename=name,
)
backend = MsWordDocumentBackend(
in_doc=in_doc,
path_or_stream=path,
)
gt_cells: list[bool] = []
# table: Table with rich cells
gt_cells.extend([False, False, True, True, True, True, True, False])
# table: Table with nested table
gt_cells.extend([False, False, False, True, True, True])
# table: Table with pictures
gt_cells.extend([False, False, False, True, True, False])
gt_it = iter(gt_cells)
for idx_t, table in enumerate(backend.docx_obj.tables):
for idx_r, row in enumerate(table.rows):
for idx_c, cell in enumerate(row.cells):
assert next(gt_it) == backend._is_rich_table_cell(cell), (
f"Wrong cell type in table {idx_t}, row {idx_r}, col {idx_c} "
f"with text: {cell.text}"
)