feat!: Docling v2 (#117)

---------

Signed-off-by: Christoph Auer <cau@zurich.ibm.com>
Signed-off-by: Maxim Lysak <mly@zurich.ibm.com>
Signed-off-by: Michele Dolfi <dol@zurich.ibm.com>
Signed-off-by: Panos Vagenas <35837085+vagenas@users.noreply.github.com>
Co-authored-by: Maxim Lysak <mly@zurich.ibm.com>
Co-authored-by: Michele Dolfi <dol@zurich.ibm.com>
Co-authored-by: Panos Vagenas <35837085+vagenas@users.noreply.github.com>

docs/examples/run_with_formats.py

@@ -0,0 +1,76 @@
+import json
+import logging
+from pathlib import Path
+
+import yaml
+
+from docling.backend.pypdfium2_backend import PyPdfiumDocumentBackend
+from docling.datamodel.base_models import InputFormat
+from docling.document_converter import (
+    DocumentConverter,
+    PdfFormatOption,
+    WordFormatOption,
+)
+from docling.pipeline.simple_pipeline import SimplePipeline
+from docling.pipeline.standard_pdf_pipeline import StandardPdfPipeline
+
+_log = logging.getLogger(__name__)
+
+
+def main():
+    input_paths = [
+        Path("tests/data/wiki_duck.html"),
+        Path("tests/data/word_sample.docx"),
+        Path("tests/data/lorem_ipsum.docx"),
+        Path("tests/data/powerpoint_sample.pptx"),
+        Path("tests/data/2305.03393v1-pg9-img.png"),
+        Path("tests/data/2206.01062.pdf"),
+    ]
+
+    ## for defaults use:
+    # doc_converter = DocumentConverter()
+
+    ## to customize use:
+
+    doc_converter = (
+        DocumentConverter(  # all of the below is optional, has internal defaults.
+            allowed_formats=[
+                InputFormat.PDF,
+                InputFormat.IMAGE,
+                InputFormat.DOCX,
+                InputFormat.HTML,
+                InputFormat.PPTX,
+            ],  # whitelist formats, non-matching files are ignored.
+            format_options={
+                InputFormat.PDF: PdfFormatOption(
+                    pipeline_cls=StandardPdfPipeline, backend=PyPdfiumDocumentBackend
+                ),
+                InputFormat.DOCX: WordFormatOption(
+                    pipeline_cls=SimplePipeline  # , backend=MsWordDocumentBackend
+                ),
+            },
+        )
+    )
+
+    conv_results = doc_converter.convert_all(input_paths)
+
+    for res in conv_results:
+        out_path = Path("scratch")
+        print(
+            f"Document {res.input.file.name} converted."
+            f"\nSaved markdown output to: {str(out_path)}"
+        )
+        # print(res.docdocument.export_to_markdown())
+        # Export Docling document format to markdowndoc:
+        with (out_path / f"{res.input.file.name}.md").open("w") as fp:
+            fp.write(res.document.export_to_markdown())
+
+        with (out_path / f"{res.input.file.name}.json").open("w") as fp:
+            fp.write(json.dumps(res.document.export_to_dict()))
+
+        with (out_path / f"{res.input.file.name}.yaml").open("w") as fp:
+            fp.write(yaml.safe_dump(res.document.export_to_dict()))
+
+
+if __name__ == "__main__":
+    main()