diff --git a/poetry.lock b/poetry.lock index e7b941b6..6c0017af 100644 --- a/poetry.lock +++ b/poetry.lock @@ -887,7 +887,9 @@ pandas = ">=2.1.4,<3.0.0" pillow = ">=10.0.0,<12.0.0" pydantic = ">=2.6.0,<2.10.0 || >2.10.0,<2.10.1 || >2.10.1,<2.10.2 || >2.10.2,<3.0.0" pyyaml = ">=5.1,<7.0.0" +semchunk = {version = ">=2.2.0,<3.0.0", optional = true, markers = "extra == \"chunking\""} tabulate = ">=0.9.0,<0.10.0" +transformers = {version = ">=4.34.0,<5.0.0", optional = true, markers = "extra == \"chunking\""} typer = ">=0.12.5,<0.13.0" typing-extensions = ">=4.12.2,<5.0.0" @@ -2826,6 +2828,32 @@ files = [ {file = "more_itertools-10.6.0-py3-none-any.whl", hash = "sha256:6eb054cb4b6db1473f6e15fcc676a08e4732548acd47c708f0e179c2c7c01e89"}, ] +[[package]] +name = "mpire" +version = "2.10.2" +description = "A Python package for easy multiprocessing, but faster than multiprocessing" +optional = false +python-versions = "*" +files = [ + {file = "mpire-2.10.2-py3-none-any.whl", hash = "sha256:d627707f7a8d02aa4c7f7d59de399dec5290945ddf7fbd36cbb1d6ebb37a51fb"}, + {file = "mpire-2.10.2.tar.gz", hash = "sha256:f66a321e93fadff34585a4bfa05e95bd946cf714b442f51c529038eb45773d97"}, +] + +[package.dependencies] +multiprocess = [ + {version = "*", optional = true, markers = "python_version < \"3.11\" and extra == \"dill\""}, + {version = ">=0.70.15", optional = true, markers = "python_version >= \"3.11\" and extra == \"dill\""}, +] +pygments = ">=2.0" +pywin32 = {version = ">=301", markers = "platform_system == \"Windows\""} +tqdm = ">=4.27" + +[package.extras] +dashboard = ["flask"] +dill = ["multiprocess", "multiprocess (>=0.70.15)"] +docs = ["docutils (==0.17.1)", "sphinx (==3.2.1)", "sphinx-autodoc-typehints (==1.11.0)", "sphinx-rtd-theme (==0.5.0)", "sphinx-versions (==1.0.1)", "sphinxcontrib-images (==0.9.2)"] +testing = ["ipywidgets", "multiprocess", "multiprocess (>=0.70.15)", "numpy", "pywin32 (>=301)", "rich"] + [[package]] name = "mpmath" version = "1.3.0" @@ -6133,6 +6161,21 @@ files = [ cryptography = ">=2.0" jeepney = ">=0.6" +[[package]] +name = "semchunk" +version = "2.2.2" +description = "A fast and lightweight Python library for splitting text into semantically meaningful chunks." +optional = false +python-versions = ">=3.9" +files = [ + {file = "semchunk-2.2.2-py3-none-any.whl", hash = "sha256:94ca19020c013c073abdfd06d79a7c13637b91738335f3b8cdb5655ee7cc94d2"}, + {file = "semchunk-2.2.2.tar.gz", hash = "sha256:940e89896e64eeb01de97ba60f51c8c7b96c6a3951dfcf574f25ce2146752f52"}, +] + +[package.dependencies] +mpire = {version = "*", extras = ["dill"]} +tqdm = "*" + [[package]] name = "semver" version = "2.13.0" @@ -7816,4 +7859,4 @@ vlm = ["accelerate", "transformers", "transformers"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "99d99a9de4a1826363c11484687c2b69cbe8b6430684277d61f9207d8cb116f7" +content-hash = "a9830e60f15bd80e7776c5dbba86dd477ae62408e55406c3f91a127bc553e173" diff --git a/pyproject.toml b/pyproject.toml index 012b71e9..abd0365c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ packages = [{ include = "docling" }] ###################### python = "^3.9" pydantic = "^2.0.0" -docling-core = "^2.26.0" +docling-core = {version = "^2.26.0", extras = ["chunking"]} docling-ibm-models = "^3.4.0" docling-parse = "^4.0.0" filetype = "^1.2.0"