diff --git a/docling/document_converter.py b/docling/document_converter.py index d52efcea..42288cca 100644 --- a/docling/document_converter.py +++ b/docling/document_converter.py @@ -5,6 +5,7 @@ import time from functools import partial from pathlib import Path from typing import Dict, Iterable, Iterator, List, Optional, Tuple, Type, Union +import hashlib from pydantic import BaseModel, ConfigDict, model_validator, validate_call @@ -181,7 +182,12 @@ class DocumentConverter: ) for format in self.allowed_formats } - self.initialized_pipelines: Dict[Type[BasePipeline], BasePipeline] = {} + self.initialized_pipelines: Dict[Tuple[Type[BasePipeline], str], BasePipeline] = {} + + def _get_pipeline_options_hash(self, pipeline_options: PipelineOptions) -> str: + """Generate a hash of pipeline options to use as part of the cache key.""" + options_str = str(pipeline_options.model_dump() if hasattr(pipeline_options, 'model_dump') else pipeline_options) + return hashlib.md5(options_str.encode('utf-8')).hexdigest() def initialize_pipeline(self, format: InputFormat): """Initialize the conversion pipeline for the selected format.""" @@ -279,31 +285,30 @@ class DocumentConverter: yield item def _get_pipeline(self, doc_format: InputFormat) -> Optional[BasePipeline]: + """Retrieve or initialize a pipeline, reusing instances based on class and options.""" fopt = self.format_to_options.get(doc_format) - if fopt is None: + if fopt is None or fopt.pipeline_options is None: return None - else: - pipeline_class = fopt.pipeline_cls - pipeline_options = fopt.pipeline_options - if pipeline_options is None: - return None - # TODO this will ignore if different options have been defined for the same pipeline class. - if ( - pipeline_class not in self.initialized_pipelines - or self.initialized_pipelines[pipeline_class].pipeline_options - != pipeline_options - ): - self.initialized_pipelines[pipeline_class] = pipeline_class( - pipeline_options=pipeline_options - ) - return self.initialized_pipelines[pipeline_class] + pipeline_class = fopt.pipeline_cls + pipeline_options = fopt.pipeline_options + options_hash = self._get_pipeline_options_hash(pipeline_options) + + # Use a composite key to cache pipelines + cache_key = (pipeline_class, options_hash) + + if cache_key not in self.initialized_pipelines: + _log.info(f"Initializing pipeline for {pipeline_class.__name__} with options hash {options_hash}") + self.initialized_pipelines[cache_key] = pipeline_class(pipeline_options=pipeline_options) + else: + _log.debug(f"Reusing cached pipeline for {pipeline_class.__name__} with options hash {options_hash}") + + return self.initialized_pipelines[cache_key] def _process_document( self, in_doc: InputDocument, raises_on_error: bool ) -> ConversionResult: - valid = ( self.allowed_formats is not None and in_doc.format in self.allowed_formats ) @@ -345,7 +350,6 @@ class DocumentConverter: else: if raises_on_error: raise ConversionError(f"Input document {in_doc.file} is not valid.") - else: # invalid doc or not of desired format conv_res = ConversionResult(