mirrors/docling
1
0
Fork 0
mirror of https://github.com/DS4SD/docling.git synced 2025-07-31 14:34:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

reformatted the text

Browse Source 
Signed-off-by: Peter Staar <taa@zurich.ibm.com>
This commit is contained in:
Peter Staar 2024-10-29 06:02:49 +01:00
parent 7cb7da7ce9
commit 6163409305
3 changed files with 67 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
tests/test_backend_html.py
View File

@ -1,16 +1,19 @@
import os
import json import json
import os
from pathlib import Path from pathlib import Path
from docling.backend.html_backend import HTMLDocumentBackend from docling.backend.html_backend import HTMLDocumentBackend
from docling.datamodel.base_models import InputFormat from docling.datamodel.base_models import InputFormat
from docling.datamodel.document import InputDocument, SectionHeaderItem from docling.datamodel.document import (
from docling.datamodel.document import ConversionResult ConversionResult,
InputDocument,
SectionHeaderItem,
)
from docling.document_converter import DocumentConverter from docling.document_converter import DocumentConverter
GENERATE = False GENERATE = False
def test_heading_levels(): def test_heading_levels():
in_path = Path("tests/data/html/wiki_duck.html") in_path = Path("tests/data/html/wiki_duck.html")
in_doc = InputDocument( in_doc = InputDocument(
@ -35,6 +38,7 @@ def test_heading_levels():
assert item.level == 3 assert item.level == 3
assert found_lvl_2 and found_lvl_3 assert found_lvl_2 and found_lvl_3
def get_html_paths(): def get_html_paths():
# Define the directory you want to search # Define the directory you want to search
@ -44,14 +48,14 @@ def get_html_paths():
html_files = sorted(directory.rglob("*.html")) html_files = sorted(directory.rglob("*.html"))
return html_files return html_files
def get_converter(): def get_converter():
converter = DocumentConverter( converter = DocumentConverter(allowed_formats=[InputFormat.HTML])
allowed_formats = [InputFormat.HTML]
)
return converter return converter
def verify_export(pred_text: str, gtfile: str): def verify_export(pred_text: str, gtfile: str):
if not os.path.exists(gtfile) or GENERATE: if not os.path.exists(gtfile) or GENERATE:
@ -67,6 +71,7 @@ def verify_export(pred_text:str, gtfile:str):
assert pred_text == true_text, "pred_itxt==true_itxt" assert pred_text == true_text, "pred_itxt==true_itxt"
return pred_text == true_text return pred_text == true_text
def test_e2e_html_conversions(): def test_e2e_html_conversions():
html_paths = get_html_paths() html_paths = get_html_paths()
@ -82,10 +87,12 @@ def test_e2e_html_conversions():
pred_md: str = doc.export_to_markdown() pred_md: str = doc.export_to_markdown()
assert verify_export(pred_md, str(html_path) + ".md"), "export to md" assert verify_export(pred_md, str(html_path) + ".md"), "export to md"
pred_itxt:str = doc._export_to_indented_text(max_text_len=70, explicit_tables=False) pred_itxt: str = doc._export_to_indented_text(
assert verify_export(pred_itxt, str(html_path)+".itxt"), "export to indented-text" max_text_len=70, explicit_tables=False
)
assert verify_export(
pred_itxt, str(html_path) + ".itxt"
), "export to indented-text"
pred_json: str = json.dumps(doc.export_to_dict(), indent=2) pred_json: str = json.dumps(doc.export_to_dict(), indent=2)
assert verify_export(pred_json, str(html_path) + ".json"), "export to json" assert verify_export(pred_json, str(html_path) + ".json"), "export to json"

26
tests/test_backend_msword.py
View File

@ -1,16 +1,19 @@
import os
import json import json
import os
from pathlib import Path from pathlib import Path
from docling.backend.msword_backend import MsWordDocumentBackend from docling.backend.msword_backend import MsWordDocumentBackend
from docling.datamodel.base_models import InputFormat from docling.datamodel.base_models import InputFormat
from docling.datamodel.document import InputDocument, SectionHeaderItem from docling.datamodel.document import (
from docling.datamodel.document import ConversionResult ConversionResult,
InputDocument,
SectionHeaderItem,
)
from docling.document_converter import DocumentConverter from docling.document_converter import DocumentConverter
GENERATE = False GENERATE = False
def test_heading_levels(): def test_heading_levels():
in_path = Path("tests/data/docx/word_sample.docx") in_path = Path("tests/data/docx/word_sample.docx")
in_doc = InputDocument( in_doc = InputDocument(
@ -45,14 +48,14 @@ def get_docx_paths():
pdf_files = sorted(directory.rglob("*.docx")) pdf_files = sorted(directory.rglob("*.docx"))
return pdf_files return pdf_files
def get_converter(): def get_converter():
converter = DocumentConverter( converter = DocumentConverter(allowed_formats=[InputFormat.DOCX])
allowed_formats = [InputFormat.DOCX]
)
return converter return converter
def verify_export(pred_text: str, gtfile: str): def verify_export(pred_text: str, gtfile: str):
if not os.path.exists(gtfile) or GENERATE: if not os.path.exists(gtfile) or GENERATE:
@ -84,9 +87,12 @@ def test_e2e_docx_conversions():
pred_md: str = doc.export_to_markdown() pred_md: str = doc.export_to_markdown()
assert verify_export(pred_md, str(docx_path) + ".md"), "export to md" assert verify_export(pred_md, str(docx_path) + ".md"), "export to md"
pred_itxt:str = doc._export_to_indented_text(max_text_len=70, explicit_tables=False) pred_itxt: str = doc._export_to_indented_text(
assert verify_export(pred_itxt, str(docx_path)+".itxt"), "export to indented-text" max_text_len=70, explicit_tables=False
)
assert verify_export(
pred_itxt, str(docx_path) + ".itxt"
), "export to indented-text"
pred_json: str = json.dumps(doc.export_to_dict(), indent=2) pred_json: str = json.dumps(doc.export_to_dict(), indent=2)
assert verify_export(pred_json, str(docx_path) + ".json"), "export to json" assert verify_export(pred_json, str(docx_path) + ".json"), "export to json"