mirror of
https://github.com/DS4SD/docling.git
synced 2025-12-08 20:58:11 +00:00
fix(xlsx): speed up by detecting the true last non-empty row/column (#2404)
* Update msexcel_backend.py Fix #2307, Follow the instruction of https://github.com/docling-project/docling/issues/2307#issuecomment-3327248503. Signed-off-by: Richard (Huangrui) Chu <65276824+HuangruiChu@users.noreply.github.com> * Update msexcel_backend.py Fix error Signed-off-by: Richard (Huangrui) Chu <65276824+HuangruiChu@users.noreply.github.com> * Fix linting issues Signed-off-by: Richard (Huangrui) Chu <65276824+HuangruiChu@users.noreply.github.com> * Add test files and data (Signed-off-by: Huangrui Chu <huangrui.chu.1999@gmail.com>) Signed-off-by: Richard (Huangrui) Chu <65276824+HuangruiChu@users.noreply.github.com> * resolve conflict with test_backend_msexecl; update the boundary Signed-off-by: Richard (Huangrui) Chu <65276824+HuangruiChu@users.noreply.github.com> * chore(xlsx): use a dataclass to represent a bounding rectangle in worksheets Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * chore(xlsx): increase parsing speed by iterating on 'sheet._cells' Increase the parsing speed of the spreadsheet backend by iterating on 'sheets._cells' since this is proportional to the number of created cells. Rename test file to align it to other test files. Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> --------- Signed-off-by: Richard (Huangrui) Chu <65276824+HuangruiChu@users.noreply.github.com> Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> Co-authored-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
This commit is contained in:
committed by
GitHub
parent
657ce8b01c
commit
b66624bfff
@@ -2,6 +2,7 @@ import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from openpyxl import load_workbook
|
||||
|
||||
from docling.backend.msexcel_backend import MsExcelDocumentBackend
|
||||
from docling.datamodel.base_models import InputFormat
|
||||
@@ -113,3 +114,67 @@ def test_chartsheet(documents) -> None:
|
||||
assert doc.groups[1].name == "sheet: Duck Chart"
|
||||
assert doc.pages[2].size.height == 0
|
||||
assert doc.pages[2].size.width == 0
|
||||
|
||||
|
||||
def test_inflated_rows_handling(documents) -> None:
|
||||
"""Test that files with inflated max_row are handled correctly.
|
||||
|
||||
xlsx_04_inflated.xlsx has inflated max_row (1,048,496) but only 7 rows of actual data.
|
||||
This test verifies that our backend correctly identifies true data bounds.
|
||||
"""
|
||||
# First, verify the file has inflated max_row using openpyxl directly
|
||||
path = next(item for item in get_excel_paths() if item.stem == "xlsx_04_inflated")
|
||||
|
||||
wb = load_workbook(path)
|
||||
ws = wb.active
|
||||
reported_max_row = ws.max_row
|
||||
|
||||
# Assert that openpyxl reports inflated max_row
|
||||
assert reported_max_row > 100000, (
|
||||
f"xlsx_04_inflated.xlsx should have inflated max_row (expected >100k, got {reported_max_row:,}). "
|
||||
f"This test file is designed to verify proper handling of Excel files with inflated row counts."
|
||||
)
|
||||
|
||||
_log.info(
|
||||
f"xlsx_04_inflated.xlsx - Openpyxl reported max_row: {reported_max_row:,}"
|
||||
)
|
||||
|
||||
# Now test that our backend handles it correctly
|
||||
in_doc = InputDocument(
|
||||
path_or_stream=path,
|
||||
format=InputFormat.XLSX,
|
||||
filename=path.stem,
|
||||
backend=MsExcelDocumentBackend,
|
||||
)
|
||||
backend = MsExcelDocumentBackend(in_doc=in_doc, path_or_stream=path)
|
||||
|
||||
# Verify backend detects correct number of pages (should be 4, like test-01)
|
||||
page_count = backend.page_count()
|
||||
assert page_count == 4, (
|
||||
f"Backend should detect 4 pages (same as test-01), got {page_count}"
|
||||
)
|
||||
|
||||
# Verify converted document has correct pages
|
||||
doc = next(item for path, item in documents if path.stem == "xlsx_04_inflated")
|
||||
assert len(doc.pages) == 4, f"Document should have 4 pages, got {len(doc.pages)}"
|
||||
|
||||
# Verify page sizes match expected dimensions (same as test-01)
|
||||
# These should reflect actual data, not inflated row counts
|
||||
assert doc.pages.get(1).size.as_tuple() == (3.0, 7.0), (
|
||||
f"Page 1 should be 3x7 cells, got {doc.pages.get(1).size.as_tuple()}"
|
||||
)
|
||||
assert doc.pages.get(2).size.as_tuple() == (9.0, 18.0), (
|
||||
f"Page 2 should be 9x18 cells, got {doc.pages.get(2).size.as_tuple()}"
|
||||
)
|
||||
assert doc.pages.get(3).size.as_tuple() == (13.0, 36.0), (
|
||||
f"Page 3 should be 13x36 cells, got {doc.pages.get(3).size.as_tuple()}"
|
||||
)
|
||||
assert doc.pages.get(4).size.as_tuple() == (0.0, 0.0), (
|
||||
f"Page 4 should be 0x0 cells (empty), got {doc.pages.get(4).size.as_tuple()}"
|
||||
)
|
||||
|
||||
_log.info(
|
||||
f"✓ Successfully handled inflated max_row: "
|
||||
f"reported {reported_max_row:,} rows, "
|
||||
f"correctly processed as {page_count} pages with proper dimensions"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user