mirror of
https://github.com/DS4SD/docling.git
synced 2025-07-27 04:24:45 +00:00
feat(dolphin): add dolphin support
Signed-off-by: Georg Heiler <georg.kf.heiler@gmail.com>
This commit is contained in:
parent
7bae3b6c06
commit
1c3699eaf7
@ -27,6 +27,7 @@ class TransformersModelType(str, Enum):
|
|||||||
AUTOMODEL = "automodel"
|
AUTOMODEL = "automodel"
|
||||||
AUTOMODEL_VISION2SEQ = "automodel-vision2seq"
|
AUTOMODEL_VISION2SEQ = "automodel-vision2seq"
|
||||||
AUTOMODEL_CAUSALLM = "automodel-causallm"
|
AUTOMODEL_CAUSALLM = "automodel-causallm"
|
||||||
|
AUTOMODEL_FORIMAGETEXTTOTEXT = "automodel-forimagetexttotext"
|
||||||
|
|
||||||
|
|
||||||
class InlineVlmOptions(BaseVlmOptions):
|
class InlineVlmOptions(BaseVlmOptions):
|
||||||
|
@ -42,6 +42,7 @@ class HuggingFaceTransformersVlmModel(BasePageModel, HuggingFaceModelDownloadMix
|
|||||||
AutoModel,
|
AutoModel,
|
||||||
AutoModelForCausalLM,
|
AutoModelForCausalLM,
|
||||||
AutoModelForVision2Seq,
|
AutoModelForVision2Seq,
|
||||||
|
AutoModelForImageTextToText,
|
||||||
AutoProcessor,
|
AutoProcessor,
|
||||||
BitsAndBytesConfig,
|
BitsAndBytesConfig,
|
||||||
GenerationConfig,
|
GenerationConfig,
|
||||||
@ -91,6 +92,9 @@ class HuggingFaceTransformersVlmModel(BasePageModel, HuggingFaceModelDownloadMix
|
|||||||
== TransformersModelType.AUTOMODEL_VISION2SEQ
|
== TransformersModelType.AUTOMODEL_VISION2SEQ
|
||||||
):
|
):
|
||||||
model_cls = AutoModelForVision2Seq
|
model_cls = AutoModelForVision2Seq
|
||||||
|
elif (self.vlm_options.transformers_model_type
|
||||||
|
== TransformersModelType.AUTOMODEL_FORIMAGETEXTTOTEXT):
|
||||||
|
model_cls = AutoModelForImageTextToText
|
||||||
|
|
||||||
self.processor = AutoProcessor.from_pretrained(
|
self.processor = AutoProcessor.from_pretrained(
|
||||||
artifacts_path,
|
artifacts_path,
|
||||||
@ -175,6 +179,12 @@ class HuggingFaceTransformersVlmModel(BasePageModel, HuggingFaceModelDownloadMix
|
|||||||
_log.debug(f"prompt for {self.vlm_options.repo_id}: {prompt}")
|
_log.debug(f"prompt for {self.vlm_options.repo_id}: {prompt}")
|
||||||
|
|
||||||
return prompt
|
return prompt
|
||||||
|
if self.vlm_options.repo_id.lower().startswith("bytedance/dolphin"):
|
||||||
|
_log.debug("Using specialized prompt for dolphin")
|
||||||
|
# more info here https://huggingface.co/ByteDance/Dolphin
|
||||||
|
prompt = f"<s>{self.vlm_options.prompt} <Answer/>"
|
||||||
|
_log.debug(f"prompt for {self.vlm_options.repo_id}: {prompt}")
|
||||||
|
return prompt
|
||||||
|
|
||||||
messages = [
|
messages = [
|
||||||
{
|
{
|
||||||
|
Loading…
Reference in New Issue
Block a user