propagate core update

Signed-off-by: Panos Vagenas <pva@zurich.ibm.com>

poetry.lock

@@ -897,8 +897,8 @@ chunking = ["semchunk (>=2.2.0,<3.0.0)", "transformers (>=4.34.0,<5.0.0)"]
 [package.source]
 type = "git"
 url = "https://github.com/docling-project/docling-core.git"
-reference = "80704dce4b21656b50bf04182d4636f448c49827"
-resolved_reference = "80704dce4b21656b50bf04182d4636f448c49827"
+reference = "30c2e87536fb2fded0170baed39b4d001d620823"
+resolved_reference = "30c2e87536fb2fded0170baed39b4d001d620823"
 
 [[package]]
 name = "docling-ibm-models"
@@ -7842,4 +7842,4 @@ vlm = ["accelerate", "transformers", "transformers"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "891d40e03852f5de1979c76134f1ea6b4876abf6cb14a27c3c22f4aa3d0e37f4"
+content-hash = "356cabb283e4d64913d3345d8b387a47f38ed69f358914112e2b1f11c498d999"

pyproject.toml

@@ -47,7 +47,7 @@ packages = [{ include = "docling" }]
 python = "^3.9"
 pydantic = "^2.0.0"
 # docling-core = { git = "https://github.com/docling-project/docling-core.git", extras = ["chunking"], branch = "main" }
-docling-core = { git = "https://github.com/docling-project/docling-core.git", extras = ["chunking"], rev = "80704dce4b21656b50bf04182d4636f448c49827" }
+docling-core = { git = "https://github.com/docling-project/docling-core.git", extras = ["chunking"], rev = "30c2e87536fb2fded0170baed39b4d001d620823" }
 docling-ibm-models = "^3.4.0"
 docling-parse = "^4.0.0"
 filetype = "^1.2.0"

tests/data/groundtruth/docling_v2/2203.01017v2.doctags.txt

@@ -229,6 +229,7 @@
 <otsl><loc_68><loc_148><loc_195><loc_184></otsl>
 <otsl><loc_69><loc_195><loc_195><loc_232></otsl>
 <otsl><loc_68><loc_250><loc_203><loc_308></otsl>
+<caption><loc_52><loc_317><loc_223><loc_323>Figure 8: Example of a table with multi-line header.</caption>
 <otsl><loc_254><loc_64><loc_454><loc_86><caption><loc_252><loc_194><loc_445><loc_207>Figure 9: Example of a table with big empty distance between cells.</caption></otsl>
 <otsl><loc_253><loc_98><loc_454><loc_117></otsl>
 <otsl><loc_253><loc_124><loc_454><loc_147></otsl>
@@ -243,10 +244,12 @@
 <page_break>
 <otsl><loc_42><loc_173><loc_231><loc_217></otsl>
 <picture><loc_42><loc_66><loc_231><loc_218></picture>
+<caption><loc_41><loc_225><loc_234><loc_238>Figure 11: Simple table with different style and empty cells.</caption>
 <otsl><loc_42><loc_286><loc_254><loc_310></otsl>
 <otsl><loc_42><loc_318><loc_254><loc_342></otsl>
 <otsl><loc_42><loc_350><loc_254><loc_374></otsl>
 <picture><loc_41><loc_386><loc_145><loc_414></picture>
+<caption><loc_45><loc_424><loc_230><loc_430>Figure 12: Simple table predictions and post processing.</caption>
 <otsl><loc_261><loc_102><loc_437><loc_135></otsl>
 <otsl><loc_261><loc_143><loc_437><loc_177></otsl>
 <otsl><loc_268><loc_182><loc_428><loc_226></otsl>
@@ -270,6 +273,7 @@
 <picture><loc_289><loc_308><loc_405><loc_401></picture>
 <otsl><loc_289><loc_308><loc_405><loc_401></otsl>
 <otsl><loc_41><loc_319><loc_261><loc_399></otsl>
+<caption><loc_252><loc_412><loc_445><loc_425>Figure 16: Example of how post-processing helps to restore mis-aligned bounding boxes prediction artifact.</caption>
 <page_footer><loc_239><loc_463><loc_247><loc_469>15</page_footer>
 <page_break>
 <picture><loc_55><loc_160><loc_432><loc_314><caption><loc_41><loc_321><loc_445><loc_334>Figure 17: Example of long table. End-to-end example from initial PDF cells to prediction of bounding boxes, post processing and prediction of structure.</caption></picture>

tests/data/groundtruth/docling_v2/2203.01017v2.md

@@ -395,6 +395,8 @@ phan cell.
 
 Aditional images with examples of TableFormer predictions and post-processing can be found below.
 
+Figure 8: Example of a table with multi-line header.
+
 Figure 9: Example of a table with big empty distance between cells.
 
 <!-- image -->
@@ -405,8 +407,12 @@ Figure 10: Example of a complex table with empty cells.
 
 <!-- image -->
 
+Figure 11: Simple table with different style and empty cells.
+
 <!-- image -->
 
+Figure 12: Simple table predictions and post processing.
+
 Figure 13: Table predictions example on colorful table.
 
 <!-- image -->
@@ -427,6 +433,8 @@ Figure 15: Example with triangular table.
 
 <!-- image -->
 
+Figure 16: Example of how post-processing helps to restore mis-aligned bounding boxes prediction artifact.
+
 Figure 17: Example of long table. End-to-end example from initial PDF cells to prediction of bounding boxes, post processing and prediction of structure.
 
 <!-- image -->

tests/data/groundtruth/docling_v2/2206.01062.doctags.txt

@@ -82,6 +82,7 @@
 <text><loc_44><loc_364><loc_241><loc_446>Phase 3: Training. After a first trial with a small group of people, we realised that providing the annotation guideline and a set of random practice pages did not yield the desired quality level for layout annotation. Therefore we prepared a subset of pages with two different complexity levels, each with a practice and an exam part. 974 pages were reference-annotated by one proficient core team member. Annotation staff were then given the task to annotate the same subsets (blinded from the reference). By comparing the annotations of each staff member with the reference annotations, we could quantify how closely their annotations matched the reference. Only after passing two exam levels with high annotation quality, staff were admitted into the production phase. Practice iterations</text>
 <picture><loc_258><loc_54><loc_457><loc_290></picture>
 <text><loc_327><loc_289><loc_389><loc_291>05237a14f2524e3f53c8454b074409d05078038a6a36b770fcc8ec7e540deae0</text>
+<caption><loc_260><loc_299><loc_457><loc_318>Figure 4: Examples of plausible annotation alternatives for the same page. Criteria in our annotation guideline can resolve cases A to C, while the case D remains ambiguous.</caption>
 <text><loc_259><loc_332><loc_456><loc_344>were carried out over a timeframe of 12 weeks, after which 8 of the 40 initially allocated annotators did not pass the bar.</text>
 <text><loc_259><loc_346><loc_457><loc_448>Phase 4: Production annotation. The previously selected 80K pages were annotated with the defined 11 class labels by 32 annotators. This production phase took around three months to complete. All annotations were created online through CCS, which visualises the programmatic PDF text-cells as an overlay on the page. The page annotation are obtained by drawing rectangular bounding-boxes, as shown in Figure 3. With regard to the annotation practices, we implemented a few constraints and capabilities on the tooling level. First, we only allow non-overlapping, vertically oriented, rectangular boxes. For the large majority of documents, this constraint was sufficient and it speeds up the annotation considerably in comparison with arbitrary segmentation shapes. Second, annotator staff were not able to see each other's annotations. This was enforced by design to avoid any bias in the annotation, which could skew the numbers of the inter-annotator agreement (see Table 1). We wanted</text>
 <page_break>

tests/data/groundtruth/docling_v2/2206.01062.md

@@ -152,6 +152,8 @@ Phase 3: Training. After a first trial with a small group of people, we realised
 
 05237a14f2524e3f53c8454b074409d05078038a6a36b770fcc8ec7e540deae0
 
+Figure 4: Examples of plausible annotation alternatives for the same page. Criteria in our annotation guideline can resolve cases A to C, while the case D remains ambiguous.
+
 were carried out over a timeframe of 12 weeks, after which 8 of the 40 initially allocated annotators did not pass the bar.
 
 Phase 4: Production annotation. The previously selected 80K pages were annotated with the defined 11 class labels by 32 annotators. This production phase took around three months to complete. All annotations were created online through CCS, which visualises the programmatic PDF text-cells as an overlay on the page. The page annotation are obtained by drawing rectangular bounding-boxes, as shown in Figure 3. With regard to the annotation practices, we implemented a few constraints and capabilities on the tooling level. First, we only allow non-overlapping, vertically oriented, rectangular boxes. For the large majority of documents, this constraint was sufficient and it speeds up the annotation considerably in comparison with arbitrary segmentation shapes. Second, annotator staff were not able to see each other's annotations. This was enforced by design to avoid any bias in the annotation, which could skew the numbers of the inter-annotator agreement (see Table 1). We wanted

tests/data/groundtruth/docling_v2/redp5110_sampled.doctags.txt

@@ -105,6 +105,7 @@
 <text><loc_112><loc_155><loc_424><loc_168>The FUNCTION_USAGE view contains function usage configuration details. Table 2-1 describes the columns in the FUNCTION_USAGE view.</text>
 <otsl><loc_111><loc_183><loc_446><loc_279><ched>Column name<ched>Data type<ched>Description<nl><fcel>FUNCTION_ID<fcel>VARCHAR(30)<fcel>ID of the function.<nl><fcel>USER_NAME<fcel>VARCHAR(10)<fcel>Name of the user profile that has a usage setting for this  function.<nl><fcel>USAGE<fcel>VARCHAR(7)<fcel>Usage setting: GLYPH<SM590000> ALLOWED: The user profile is allowed to use the function. GLYPH<SM590000> DENIED: The user profile is not allowed to use the function.<nl><fcel>USER_TYPE<fcel>VARCHAR(5)<fcel>Type of user profile: GLYPH<SM590000> USER: The user profile is a user. GLYPH<SM590000> GROUP: The user profile is a group.<nl><caption><loc_112><loc_176><loc_232><loc_182>Table 2-1 FUNCTION_USAGE view</caption></otsl>
 <text><loc_112><loc_286><loc_447><loc_299>To discover who has authorization to define and manage RCAC, you can use the query that is shown in Example 2-1.</text>
+<caption><loc_112><loc_307><loc_378><loc_312>Example 2-1 Query to determine who has authority to define and manage RCAC</caption>
 <text><loc_112><loc_318><loc_140><loc_324>SELECT</text>
 <text><loc_149><loc_318><loc_206><loc_324>function_id,</text>
 <text><loc_136><loc_326><loc_197><loc_331>user_name,</text>
@@ -131,12 +132,14 @@
 <page_footer><loc_290><loc_477><loc_428><loc_482>Chapter 2. Roles and separation of duties</page_footer>
 <page_footer><loc_438><loc_477><loc_447><loc_482>11</page_footer>
 <page_break>
+<caption><loc_112><loc_45><loc_432><loc_59>The SQL CREATE PERMISSION statement that is shown in Figure 3-1 is used to define and initially enable or disable the row access rules.</caption>
 <picture><loc_111><loc_68><loc_446><loc_259><caption><loc_112><loc_261><loc_279><loc_267>Figure 3-1 CREATE PERMISSION SQL statement</caption></picture>
 <section_header_level_1><loc_112><loc_278><loc_176><loc_285>Column mask</section_header_level_1>
 <text><loc_112><loc_287><loc_443><loc_316>A column mask is a database object that manifests a column value access control rule for a specific column in a specific table. It uses a CASE expression that describes what you see when you access the column. For example, a teller can see only the last four digits of a tax identification number.</text>
 <page_footer><loc_282><loc_477><loc_428><loc_482>Chapter 3. Row and Column Access Control</page_footer>
 <page_footer><loc_438><loc_477><loc_447><loc_482>15</page_footer>
 <page_break>
+<caption><loc_112><loc_45><loc_337><loc_51>Table 3-1 summarizes these special registers and their values.</caption>
 <otsl><loc_110><loc_65><loc_443><loc_129><ched>Special register<ched>Corresponding value<nl><fcel>USER or SESSION_USER<fcel>The effective user of the thread excluding adopted authority.<nl><fcel>CURRENT_USER<fcel>The effective user of the thread including adopted authority. When no adopted  authority is present, this has the same value as USER.<nl><fcel>SYSTEM_USER<fcel>The authorization ID that initiated the connection.<nl><caption><loc_112><loc_59><loc_304><loc_64>Table 3-1 Special registers and their corresponding values</caption></otsl>
 <text><loc_112><loc_135><loc_440><loc_149>Figure 3-5 shows the difference in the special register values when an adopted authority is used:</text>
 <unordered_list><list_item><loc_112><loc_154><loc_336><loc_160>GLYPH<SM590000> A user connects to the server using the user profile ALICE.</list_item>

tests/data/groundtruth/docling_v2/redp5110_sampled.md

@@ -199,6 +199,8 @@ Table 2-1 FUNCTION\_USAGE view
 
 To discover who has authorization to define and manage RCAC, you can use the query that is shown in Example 2-1.
 
+Example 2-1 Query to determine who has authority to define and manage RCAC
+
 SELECT
 
 function\_id,
@@ -254,6 +256,8 @@ Table 2-2 Comparison of the different function usage IDs and *JOBCTL authority
 | MODIFY PLAN CACHE PROPERTIES procedure (currently does not check authority)    | X         |                  | X                |                  |                |
 | CHANGE PLAN CACHE SIZE procedure (currently does not check authority)          | X         |                  | X                |                  |                |
 
+The SQL CREATE PERMISSION statement that is shown in Figure 3-1 is used to define and initially enable or disable the row access rules.
+
 Figure 3-1 CREATE PERMISSION SQL statement
 
 <!-- image -->
@@ -262,6 +266,8 @@ Figure 3-1 CREATE PERMISSION SQL statement
 
 A column mask is a database object that manifests a column value access control rule for a specific column in a specific table. It uses a CASE expression that describes what you see when you access the column. For example, a teller can see only the last four digits of a tax identification number.
 
+Table 3-1 summarizes these special registers and their values.
+
 Table 3-1 Special registers and their corresponding values
 
 | Special register     | Corresponding value                                                                                                                   |