mirror of
https://github.com/DS4SD/docling.git
synced 2025-12-08 20:58:11 +00:00
fix(xlsx): speed up by detecting the true last non-empty row/column (#2404)
* Update msexcel_backend.py Fix #2307, Follow the instruction of https://github.com/docling-project/docling/issues/2307#issuecomment-3327248503. Signed-off-by: Richard (Huangrui) Chu <65276824+HuangruiChu@users.noreply.github.com> * Update msexcel_backend.py Fix error Signed-off-by: Richard (Huangrui) Chu <65276824+HuangruiChu@users.noreply.github.com> * Fix linting issues Signed-off-by: Richard (Huangrui) Chu <65276824+HuangruiChu@users.noreply.github.com> * Add test files and data (Signed-off-by: Huangrui Chu <huangrui.chu.1999@gmail.com>) Signed-off-by: Richard (Huangrui) Chu <65276824+HuangruiChu@users.noreply.github.com> * resolve conflict with test_backend_msexecl; update the boundary Signed-off-by: Richard (Huangrui) Chu <65276824+HuangruiChu@users.noreply.github.com> * chore(xlsx): use a dataclass to represent a bounding rectangle in worksheets Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> * chore(xlsx): increase parsing speed by iterating on 'sheet._cells' Increase the parsing speed of the spreadsheet backend by iterating on 'sheets._cells' since this is proportional to the number of created cells. Rename test file to align it to other test files. Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> --------- Signed-off-by: Richard (Huangrui) Chu <65276824+HuangruiChu@users.noreply.github.com> Signed-off-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com> Co-authored-by: Cesar Berrospi Ramis <ceb@zurich.ibm.com>
This commit is contained in:
committed by
GitHub
parent
657ce8b01c
commit
b66624bfff
11
tests/data/groundtruth/docling_v2/xlsx_04_inflated.xlsx.itxt
vendored
Normal file
11
tests/data/groundtruth/docling_v2/xlsx_04_inflated.xlsx.itxt
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
item-0 at level 0: unspecified: group _root_
|
||||
item-1 at level 1: section: group sheet: Sheet1
|
||||
item-2 at level 2: table with [7x3]
|
||||
item-3 at level 1: section: group sheet: Sheet2
|
||||
item-4 at level 2: table with [9x4]
|
||||
item-5 at level 2: table with [5x3]
|
||||
item-6 at level 2: table with [5x3]
|
||||
item-7 at level 1: section: group sheet: Sheet3
|
||||
item-8 at level 2: table with [7x3]
|
||||
item-9 at level 2: table with [7x3]
|
||||
item-10 at level 2: picture
|
||||
3775
tests/data/groundtruth/docling_v2/xlsx_04_inflated.xlsx.json
vendored
Normal file
3775
tests/data/groundtruth/docling_v2/xlsx_04_inflated.xlsx.json
vendored
Normal file
File diff suppressed because one or more lines are too long
53
tests/data/groundtruth/docling_v2/xlsx_04_inflated.xlsx.md
vendored
Normal file
53
tests/data/groundtruth/docling_v2/xlsx_04_inflated.xlsx.md
vendored
Normal file
@@ -0,0 +1,53 @@
|
||||
| first | second | third |
|
||||
|----------|-----------|---------|
|
||||
| 1 | 5 | 9 |
|
||||
| 2 | 4 | 6 |
|
||||
| 3 | 3 | 3 |
|
||||
| 4 | 2 | 0 |
|
||||
| 5 | 1 | -3 |
|
||||
| 6 | 0 | -6 |
|
||||
|
||||
| col-1 | col-2 | col-3 | col-4 |
|
||||
|---------|---------|---------|---------|
|
||||
| 1 | 2 | 3 | 4 |
|
||||
| 2 | 4 | 6 | 8 |
|
||||
| 3 | 6 | 9 | 12 |
|
||||
| 4 | 8 | 12 | 16 |
|
||||
| 5 | 10 | 15 | 20 |
|
||||
| 6 | 12 | 18 | 24 |
|
||||
| 7 | 14 | 21 | 28 |
|
||||
| 8 | 16 | 24 | 32 |
|
||||
|
||||
| col-1 | col-2 | col-3 |
|
||||
|---------|---------|---------|
|
||||
| 1 | 2 | 3 |
|
||||
| 2 | 4 | 6 |
|
||||
| 3 | 6 | 9 |
|
||||
| 4 | 8 | 12 |
|
||||
|
||||
| col-1 | col-2 | col-3 |
|
||||
|---------|---------|---------|
|
||||
| 1 | 2 | 3 |
|
||||
| 2 | 4 | 6 |
|
||||
| 3 | 6 | 9 |
|
||||
| 4 | 8 | 12 |
|
||||
|
||||
| first | header | header |
|
||||
|----------|----------|----------|
|
||||
| first | second | third |
|
||||
| 1 | 2 | 3 |
|
||||
| 3 | 4 | 5 |
|
||||
| 3 | 6 | 7 |
|
||||
| 8 | 9 | 9 |
|
||||
| 10 | 9 | 9 |
|
||||
|
||||
| first (f) | header (f) | header (f) |
|
||||
|-------------|--------------|--------------|
|
||||
| first (f) | second | third |
|
||||
| 1 | 2 | 3 |
|
||||
| 3 | 4 | 5 |
|
||||
| 3 | 6 | 7 |
|
||||
| 8 | 9 | 9 |
|
||||
| 10 | 9 | 9 |
|
||||
|
||||
<!-- image -->
|
||||
BIN
tests/data/xlsx/xlsx_04_inflated.xlsx
vendored
Normal file
BIN
tests/data/xlsx/xlsx_04_inflated.xlsx
vendored
Normal file
Binary file not shown.
@@ -2,6 +2,7 @@ import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from openpyxl import load_workbook
|
||||
|
||||
from docling.backend.msexcel_backend import MsExcelDocumentBackend
|
||||
from docling.datamodel.base_models import InputFormat
|
||||
@@ -113,3 +114,67 @@ def test_chartsheet(documents) -> None:
|
||||
assert doc.groups[1].name == "sheet: Duck Chart"
|
||||
assert doc.pages[2].size.height == 0
|
||||
assert doc.pages[2].size.width == 0
|
||||
|
||||
|
||||
def test_inflated_rows_handling(documents) -> None:
|
||||
"""Test that files with inflated max_row are handled correctly.
|
||||
|
||||
xlsx_04_inflated.xlsx has inflated max_row (1,048,496) but only 7 rows of actual data.
|
||||
This test verifies that our backend correctly identifies true data bounds.
|
||||
"""
|
||||
# First, verify the file has inflated max_row using openpyxl directly
|
||||
path = next(item for item in get_excel_paths() if item.stem == "xlsx_04_inflated")
|
||||
|
||||
wb = load_workbook(path)
|
||||
ws = wb.active
|
||||
reported_max_row = ws.max_row
|
||||
|
||||
# Assert that openpyxl reports inflated max_row
|
||||
assert reported_max_row > 100000, (
|
||||
f"xlsx_04_inflated.xlsx should have inflated max_row (expected >100k, got {reported_max_row:,}). "
|
||||
f"This test file is designed to verify proper handling of Excel files with inflated row counts."
|
||||
)
|
||||
|
||||
_log.info(
|
||||
f"xlsx_04_inflated.xlsx - Openpyxl reported max_row: {reported_max_row:,}"
|
||||
)
|
||||
|
||||
# Now test that our backend handles it correctly
|
||||
in_doc = InputDocument(
|
||||
path_or_stream=path,
|
||||
format=InputFormat.XLSX,
|
||||
filename=path.stem,
|
||||
backend=MsExcelDocumentBackend,
|
||||
)
|
||||
backend = MsExcelDocumentBackend(in_doc=in_doc, path_or_stream=path)
|
||||
|
||||
# Verify backend detects correct number of pages (should be 4, like test-01)
|
||||
page_count = backend.page_count()
|
||||
assert page_count == 4, (
|
||||
f"Backend should detect 4 pages (same as test-01), got {page_count}"
|
||||
)
|
||||
|
||||
# Verify converted document has correct pages
|
||||
doc = next(item for path, item in documents if path.stem == "xlsx_04_inflated")
|
||||
assert len(doc.pages) == 4, f"Document should have 4 pages, got {len(doc.pages)}"
|
||||
|
||||
# Verify page sizes match expected dimensions (same as test-01)
|
||||
# These should reflect actual data, not inflated row counts
|
||||
assert doc.pages.get(1).size.as_tuple() == (3.0, 7.0), (
|
||||
f"Page 1 should be 3x7 cells, got {doc.pages.get(1).size.as_tuple()}"
|
||||
)
|
||||
assert doc.pages.get(2).size.as_tuple() == (9.0, 18.0), (
|
||||
f"Page 2 should be 9x18 cells, got {doc.pages.get(2).size.as_tuple()}"
|
||||
)
|
||||
assert doc.pages.get(3).size.as_tuple() == (13.0, 36.0), (
|
||||
f"Page 3 should be 13x36 cells, got {doc.pages.get(3).size.as_tuple()}"
|
||||
)
|
||||
assert doc.pages.get(4).size.as_tuple() == (0.0, 0.0), (
|
||||
f"Page 4 should be 0x0 cells (empty), got {doc.pages.get(4).size.as_tuple()}"
|
||||
)
|
||||
|
||||
_log.info(
|
||||
f"✓ Successfully handled inflated max_row: "
|
||||
f"reported {reported_max_row:,} rows, "
|
||||
f"correctly processed as {page_count} pages with proper dimensions"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user