mirrors/docling
1
0
Fork 0
mirror of https://github.com/DS4SD/docling.git synced 2025-07-26 20:14:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(dolphin): add dolphin support

Browse Source 
Signed-off-by: Georg Heiler <georg.kf.heiler@gmail.com>
This commit is contained in:
Georg Heiler 2025-06-14 09:00:11 +02:00
parent 7bae3b6c06
commit 1c3699eaf7
2 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docling/datamodel/pipeline_options_vlm_model.py
View File

@ -27,6 +27,7 @@ class TransformersModelType(str, Enum):
AUTOMODEL = "automodel" AUTOMODEL = "automodel"
AUTOMODEL_VISION2SEQ = "automodel-vision2seq" AUTOMODEL_VISION2SEQ = "automodel-vision2seq"
AUTOMODEL_CAUSALLM = "automodel-causallm" AUTOMODEL_CAUSALLM = "automodel-causallm"
AUTOMODEL_FORIMAGETEXTTOTEXT = "automodel-forimagetexttotext"
class InlineVlmOptions(BaseVlmOptions): class InlineVlmOptions(BaseVlmOptions):

10
docling/models/vlm_models_inline/hf_transformers_model.py
View File

@ -42,6 +42,7 @@ class HuggingFaceTransformersVlmModel(BasePageModel, HuggingFaceModelDownloadMix
AutoModel, AutoModel,
AutoModelForCausalLM, AutoModelForCausalLM,
AutoModelForVision2Seq, AutoModelForVision2Seq,
AutoModelForImageTextToText,
AutoProcessor, AutoProcessor,
BitsAndBytesConfig, BitsAndBytesConfig,
GenerationConfig, GenerationConfig,
@ -91,6 +92,9 @@ class HuggingFaceTransformersVlmModel(BasePageModel, HuggingFaceModelDownloadMix
== TransformersModelType.AUTOMODEL_VISION2SEQ == TransformersModelType.AUTOMODEL_VISION2SEQ
): ):
model_cls = AutoModelForVision2Seq model_cls = AutoModelForVision2Seq
elif (self.vlm_options.transformers_model_type
== TransformersModelType.AUTOMODEL_FORIMAGETEXTTOTEXT):
model_cls = AutoModelForImageTextToText
self.processor = AutoProcessor.from_pretrained( self.processor = AutoProcessor.from_pretrained(
artifacts_path, artifacts_path,
@ -175,6 +179,12 @@ class HuggingFaceTransformersVlmModel(BasePageModel, HuggingFaceModelDownloadMix
_log.debug(f"prompt for {self.vlm_options.repo_id}: {prompt}") _log.debug(f"prompt for {self.vlm_options.repo_id}: {prompt}")
return prompt return prompt
if self.vlm_options.repo_id.lower().startswith("bytedance/dolphin"):
_log.debug("Using specialized prompt for dolphin")
# more info here https://huggingface.co/ByteDance/Dolphin
prompt = f"<s>{self.vlm_options.prompt} <Answer/>"
_log.debug(f"prompt for {self.vlm_options.repo_id}: {prompt}")
return prompt
messages = [ messages = [
{ {