mirrors/docling
1
0
Fork 0
mirror of https://github.com/DS4SD/docling.git synced 2025-07-25 19:44:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update threaded test

Browse Source 
Signed-off-by: Ubuntu <ubuntu@ip-172-31-30-253.eu-central-1.compute.internal>
This commit is contained in:
Ubuntu 2025-07-18 15:18:21 +00:00
parent 988db91bff
commit 89acdb5db2
1 changed files with 82 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
tests/test_threaded_pipeline.py
View File

@ -23,7 +23,15 @@ def test_threaded_pipeline_multiple_documents():
"tests/data/pdf/2206.01062.pdf",
"tests/data/pdf/2305.03393v1.pdf",
]
# test_files = [str(f) for f in Path("/home/ubuntu/datasets/flat_bench_set").rglob("*.pdf")]
do_ts = False
do_ocr = False
run_threaded = True
run_serial = True
if run_threaded:
# Threaded pipeline
threaded_converter = DocumentConverter(
format_options={
@ -34,8 +42,8 @@ def test_threaded_pipeline_multiple_documents():
table_batch_size=1,
ocr_batch_size=1,
batch_timeout_seconds=1.0,
do_table_structure=True,
do_ocr=True,
do_table_structure=do_ts,
do_ocr=do_ocr,
),
)
}
@ -56,17 +64,17 @@ def test_threaded_pipeline_multiple_documents():
del threaded_converter
print("\nMulti-document Pipeline Comparison:")
print(f"Threaded pipeline: {threaded_time:.2f} seconds")
if run_serial:
# Standard pipeline
standard_converter = DocumentConverter(
format_options={
InputFormat.PDF: PdfFormatOption(
pipeline_cls=StandardPdfPipeline,
pipeline_options=PdfPipelineOptions(
do_table_structure=True,
do_ocr=True,
do_table_structure=do_ts,
do_ocr=do_ocr,
),
)
}
@ -88,15 +96,18 @@ def test_threaded_pipeline_multiple_documents():
del standard_converter
print(f"Standard pipeline: {standard_time:.2f} seconds")
print(f"Speedup: {standard_time / threaded_time:.2f}x")
# Verify results
if run_threaded and run_serial:
assert len(standard_results) == len(threaded_results)
if run_serial:
for result in standard_results:
assert result.status == ConversionStatus.SUCCESS
if run_threaded:
for result in threaded_results:
assert result.status == ConversionStatus.SUCCESS
if run_serial and run_threaded:
# Basic content comparison
for i, (standard_result, threaded_result) in enumerate(
zip(standard_results, threaded_results)