mirrors/docling
1
0
Fork 0
mirror of https://github.com/DS4SD/docling.git synced 2025-07-30 22:14:37 +00:00
Code Issues Packages Projects Releases Wiki Activity
7a5dc3c438
docling/.actor/actor.sh
Václav Vančura 7a5dc3c438 Actor: Overhaul the implementation using official docling-serve image 
This commit completely revamps the Actor implementation with two major improvements:

1) CRITICAL CHANGE: Switch to official docling-serve image
   * Now using quay.io/ds4sd/docling-serve-cpu:latest as base image
   * Eliminates need for custom docling installation
   * Ensures compatibility with latest docling-serve features
   * Provides more reliable and consistent document processing

2) Fix Apify Actor KVS storage issues:
   * Standardize key names to follow Apify conventions:
     - Change "OUTPUT_RESULT" to "OUTPUT"
     - Change "DOCLING_LOG" to "LOG"
   * Add proper multi-stage Docker build:
     - First stage builds dependencies including apify-cli
     - Second stage uses official image and adds only necessary tools
   * Fix permission issues in Docker container:
     - Set up proper user and directory permissions
     - Create writable directories for temporary files and models
     - Configure environment variables for proper execution

3) Solve EACCES permission errors during CLI version checks:
   * Create temporary HOME directory with proper write permissions
   * Set APIFY_DISABLE_VERSION_CHECK=1 environment variable
   * Add NODE_OPTIONS="--no-warnings" to suppress update checks
   * Support --no-update-notifier CLI flag when available

4) Improve code organization and reliability:
   * Create reusable upload_to_kvs() function for all KVS operations
   * Ensure log files are uploaded before tools directory is removed
   * Set proper MIME types based on output format
   * Add detailed error reporting and proper cleanup
   * Display final output URLs for easy verification

This major refactoring significantly improves reliability and maintainability by leveraging the official docling-serve image while solving persistent permission and storage issues. The Actor now properly follows Apify standards while providing a more robust document processing pipeline.

Signed-off-by: Václav Vančura <commit@vancura.dev>
2025-03-13 10:39:30 +01:00

843 lines
37 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
# Function to upload content to the key-value store
upload_to_kvs() {
local content_file="$1"
local key_name="$2"
local content_type="$3"
local description="$4"
# Find the Apify CLI command
local apify_cmd=""
for cmd in "apify" "actor" "/usr/local/bin/apify" "/usr/bin/apify" "/opt/apify/cli/bin/apify"; do
if command -v "$cmd" &> /dev/null; then
apify_cmd="$cmd"
break
fi
done
if [ -n "$apify_cmd" ]; then
echo "Uploading $description to key-value store (key: $key_name)..."
# Create a temporary home directory with write permissions
export TMPDIR="/tmp/apify-home-${RANDOM}"
mkdir -p "$TMPDIR"
# Multiple strategies to disable version checking
export APIFY_DISABLE_VERSION_CHECK=1
export NODE_OPTIONS="--no-warnings"
export HOME="$TMPDIR" # Override home directory to writable location
# Use the --no-update-notifier flag if available
if $apify_cmd --help | grep -q "\--no-update-notifier"; then
if $apify_cmd --no-update-notifier actor:set-value "$key_name" --contentType "$content_type" < "$content_file"; then
echo "Successfully uploaded $description to key-value store"
local url="https://api.apify.com/v2/key-value-stores/${APIFY_DEFAULT_KEY_VALUE_STORE_ID}/records/$key_name"
echo "$description available at: $url"
rm -rf "$TMPDIR" 2>/dev/null || true # Clean up temp dir
return 0
fi
else
# Fall back to regular command if flag isn't available
if $apify_cmd actor:set-value "$key_name" --contentType "$content_type" < "$content_file"; then
echo "Successfully uploaded $description to key-value store"
local url="https://api.apify.com/v2/key-value-stores/${APIFY_DEFAULT_KEY_VALUE_STORE_ID}/records/$key_name"
echo "$description available at: $url"
rm -rf "$TMPDIR" 2>/dev/null || true # Clean up temp dir
return 0
fi
fi
echo "ERROR: Failed to upload $description to key-value store"
rm -rf "$TMPDIR" 2>/dev/null || true # Clean up temp dir
return 1
else
echo "ERROR: Apify CLI not found for $description upload"
return 1
fi
}
# --- Setup logging and error handling ---
LOG_FILE="/tmp/docling.log"
touch "$LOG_FILE" || {
echo "Fatal: Cannot create log file at $LOG_FILE"
exit 1
}
# Log to both console and file
exec 1> >(tee -a "$LOG_FILE")
exec 2> >(tee -a "$LOG_FILE" >&2)
# Exit codes
readonly ERR_API_UNAVAILABLE=15
readonly ERR_INVALID_INPUT=16
# --- Debug environment ---
echo "Date: $(date)"
echo "Python version: $(python --version 2>&1)"
echo "Docling-serve path: $(which docling-serve 2>/dev/null || echo 'Not found')"
echo "Working directory: $(pwd)"
# --- Setup tools ---
echo "Setting up tools..."
TOOLS_DIR="/tmp/docling-tools"
mkdir -p "$TOOLS_DIR"
# Copy tools if available
if [ -d "/build-files" ]; then
echo "Copying tools from /build-files..."
cp -r /build-files/* "$TOOLS_DIR/"
export PATH="$TOOLS_DIR/bin:$PATH"
else
echo "Warning: No build files directory found. Some tools may be unavailable."
fi
# Check OCR directories and ensure they're writable
echo "Checking OCR directory permissions..."
OCR_DIR="/opt/app-root/src/.EasyOCR"
if [ -d "$OCR_DIR" ]; then
# Test if we can write to the directory
if touch "$OCR_DIR/test_write" 2>/dev/null; then
echo "[✓] OCR directory is writable"
rm "$OCR_DIR/test_write"
else
echo "[✗] OCR directory is not writable, setting up alternative in /tmp"
# Create alternative in /tmp (which is writable)
mkdir -p "/tmp/.EasyOCR/user_network"
export EASYOCR_MODULE_PATH="/tmp/.EasyOCR"
fi
else
echo "OCR directory not found, creating in /tmp"
mkdir -p "/tmp/.EasyOCR/user_network"
export EASYOCR_MODULE_PATH="/tmp/.EasyOCR"
fi
# --- Starting the API ---
echo "Starting docling-serve API..."
# Create a dedicated working directory in /tmp (writable)
API_DIR="/tmp/docling-api"
mkdir -p "$API_DIR"
cd "$API_DIR"
echo "API working directory: $(pwd)"
# Find docling-serve executable
DOCLING_SERVE_PATH=$(which docling-serve)
echo "Docling-serve executable: $DOCLING_SERVE_PATH"
# Start the API with minimal parameters to avoid any issues
echo "Starting docling-serve API..."
"$DOCLING_SERVE_PATH" run --host 0.0.0.0 --port 5001 > "$API_DIR/docling-serve.log" 2>&1 &
API_PID=$!
echo "Started docling-serve API with PID: $API_PID"
# A more reliable wait for API startup
echo "Waiting for API to initialize..."
MAX_TRIES=30
tries=0
started=false
while [ $tries -lt $MAX_TRIES ]; do
tries=$((tries + 1))
# Check if process is still running
if ! ps -p $API_PID > /dev/null; then
echo "ERROR: docling-serve API process terminated unexpectedly after $tries seconds"
break
fi
# Check log for startup completion or errors
if grep -q "Application startup complete" "$API_DIR/docling-serve.log" 2>/dev/null; then
echo "[✓] API startup completed successfully after $tries seconds"
started=true
break
fi
if grep -q "Permission denied\|PermissionError" "$API_DIR/docling-serve.log" 2>/dev/null; then
echo "ERROR: Permission errors detected in API startup"
break
fi
# Sleep and check again
sleep 1
# Output a progress indicator every 5 seconds
if [ $((tries % 5)) -eq 0 ]; then
echo "Still waiting for API startup... ($tries/$MAX_TRIES seconds)"
fi
done
# Show log content regardless of outcome
echo "docling-serve log output so far:"
tail -n 20 "$API_DIR/docling-serve.log"
# Verify the API is running
if ! ps -p $API_PID > /dev/null; then
echo "ERROR: docling-serve API failed to start"
if [ -f "$API_DIR/docling-serve.log" ]; then
echo "Full log output:"
cat "$API_DIR/docling-serve.log"
fi
exit $ERR_API_UNAVAILABLE
fi
if [ "$started" != "true" ]; then
echo "WARNING: API process is running but startup completion was not detected"
echo "Will attempt to continue anyway..."
fi
# Try to verify API is responding at this point
echo "Verifying API responsiveness..."
(python -c "
import sys, time, socket
for i in range(5):
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(1)
result = s.connect_ex(('localhost', 5001))
if result == 0:
s.close()
print('Port 5001 is open and accepting connections')
sys.exit(0)
s.close()
except Exception as e:
pass
time.sleep(1)
print('Could not connect to API port after 5 attempts')
sys.exit(1)
" && echo "API verification succeeded") || echo "API verification failed, but continuing anyway"
# Define API endpoint
DOCLING_API_ENDPOINT="http://localhost:5001/v1alpha/convert/source"
# --- Processing document ---
echo "Starting document processing..."
echo "Reading input from Apify..."
INPUT=""
# Create directory if it doesn't exist
mkdir -p "/tmp/actor-input" || echo "Warning: Could not create /tmp/actor-input directory"
# List all possible input locations for debugging
echo "Listing potential input file locations:"
ls -la "/tmp/actor-input/" 2>/dev/null || echo "Cannot list /tmp/actor-input/"
ls -la "/input/" 2>/dev/null || echo "Cannot list /input/"
# Check multiple potential locations for input file
if [ -f "/tmp/actor-input/INPUT" ]; then
echo "Found standard Actor input file at /tmp/actor-input/INPUT"
echo "Content:"
cat "/tmp/actor-input/INPUT"
INPUT=$(cat "/tmp/actor-input/INPUT")
elif [ -f "/input/INPUT" ]; then
echo "Found Actor input file at /input/INPUT"
echo "Content:"
cat "/input/INPUT"
INPUT=$(cat "/input/INPUT")
# Fallback to environment variable
elif [ -n "$APIFY_INPUT_JSON" ]; then
echo "Using APIFY_INPUT_JSON environment variable"
INPUT="$APIFY_INPUT_JSON"
# Last resort: use test input - now defaulting to md as requested
else
echo "No input found, using test input with md format"
TEST_INPUT='{"documentUrl":"https://vancura.dev/assets/actor-test/facial-hairstyles-and-filtering-facepiece-respirators.pdf","ocr":true,"outputFormat":"md"}'
mkdir -p "/tmp/actor-input"
echo "$TEST_INPUT" > "/tmp/actor-input/INPUT"
INPUT="$TEST_INPUT"
fi
echo "Input content: $INPUT"
# Extract values from INPUT using Python
echo "Using Python to parse input..."
DOCUMENT_URL="$(echo "$INPUT" | python -c "import sys, json; print(json.load(sys.stdin).get('documentUrl', ''))")"
OUTPUT_FORMAT="$(echo "$INPUT" | python -c "import sys, json; print(json.load(sys.stdin).get('outputFormat', 'md'))")"
OCR_ENABLED="$(echo "$INPUT" | python -c "import sys, json; print(str(json.load(sys.stdin).get('ocr', True)).lower())")"
# Validate input schema should already enforce this, but double-check
if [ -z "$DOCUMENT_URL" ]; then
echo "ERROR: No document URL provided in input"
# Try to push data to Actor but don't exit if it fails
if command -v actor &> /dev/null; then
echo "Reporting missing document URL to Actor storage..."
if actor push-data "{\"status\": \"error\", \"error\": \"No document URL provided in input\"}" 2>&1; then
echo "Successfully pushed error message to Actor storage"
else
echo "Warning: Failed to push error message to Actor storage"
fi
fi
# Use default document URL for testing instead of exiting
echo "Using a default document URL for testing: https://arxiv.org/pdf/2408.09869"
DOCUMENT_URL="https://arxiv.org/pdf/2408.09869"
fi
if [ -z "$OUTPUT_FORMAT" ]; then
echo "No output format specified, defaulting to 'md'"
OUTPUT_FORMAT="md"
fi
echo "Input values: documentUrl=$DOCUMENT_URL, outputFormat=$OUTPUT_FORMAT, ocr=$OCR_ENABLED"
# Create the request JSON
REQUEST_JSON="{\"options\":{\"to_formats\":[\"$OUTPUT_FORMAT\"],\"ocr\":$OCR_ENABLED},\"http_sources\":[{\"url\":\"$DOCUMENT_URL\"}]}"
echo "$REQUEST_JSON" > "$API_DIR/request.json"
# Send the conversion request
echo "Sending conversion request to docling-serve API..."
python -c "
import json
import time
import sys
import os
import traceback
try:
# Load request data from temporary location
with open('$API_DIR/request.json', 'r') as f:
request_data = json.load(f)
print(f'Request to convert URL: {request_data[\"http_sources\"][0][\"url\"]}')
print(f'Output format: {request_data[\"options\"][\"to_formats\"][0]}')
print(f'OCR enabled: {request_data[\"options\"][\"ocr\"]}')
# Try requests first, fall back to urllib
try:
import requests
print('Using requests library for API call')
# Record start time for timing
start_time = time.time()
print(f'Starting conversion request at {time.strftime(\"%H:%M:%S\")}')
response = requests.post(
'$DOCLING_API_ENDPOINT',
json=request_data,
timeout=300 # 5 minutes timeout
)
elapsed = time.time() - start_time
print(f'Conversion request completed in {elapsed:.2f} seconds')
print(f'Response status code: {response.status_code}')
# Save the full response for debugging
with open('$API_DIR/full_response.txt', 'w') as f:
f.write(f'Status code: {response.status_code}\\n')
f.write(f'Headers: {response.headers}\\n\\n')
f.write(f'Content: {response.text[:10000]}...' if len(response.text) > 10000 else f'Content: {response.text}')
if response.status_code == 200:
with open('$API_DIR/response.json', 'w') as f:
f.write(response.text)
# Parse the response even if it's not valid JSON
try:
resp_data = response.json()
print('Successfully parsed response as JSON')
# Save detailed diagnostics about the response structure
with open('$API_DIR/response_structure.txt', 'w') as f:
f.write(f'Response keys: {list(resp_data.keys())}\\n')
if 'document' in resp_data:
f.write(f'Document keys: {list(resp_data[\"document\"].keys() if resp_data[\"document\"] else [])}\\n')
# Check for specific content fields with null safety
doc = resp_data['document'] or {}
if 'html_content' in doc and doc['html_content']:
f.write(f'HTML content length: {len(doc[\"html_content\"])}\\n')
elif 'html_content' in doc:
f.write('HTML content is present but empty or null\\n')
if 'md_content' in doc and doc['md_content']:
f.write(f'Markdown content length: {len(doc[\"md_content\"])}\\n')
elif 'md_content' in doc:
f.write('Markdown content is present but empty or null\\n')
if 'text_content' in doc and doc['text_content']:
f.write(f'Text content length: {len(doc[\"text_content\"])}\\n')
elif 'text_content' in doc:
f.write('Text content is present but empty or null\\n')
if 'json_content' in doc and doc['json_content']:
f.write(f'JSON content length: {len(doc[\"json_content\"])}\\n')
elif 'json_content' in doc:
f.write('JSON content is present but empty or null\\n')
if 'outputs' in resp_data:
f.write(f'Outputs count: {len(resp_data[\"outputs\"])}\\n')
if resp_data['outputs']:
f.write(f'First output keys: {list(resp_data[\"outputs\"][0].keys())}\\n')
if 'files' in resp_data['outputs'][0]:
f.write(f'Files count: {len(resp_data[\"outputs\"][0][\"files\"])}\\n')
if resp_data['outputs'][0]['files']:
f.write(f'First file keys: {list(resp_data[\"outputs\"][0][\"files\"][0].keys())}\\n')
if 'content' in resp_data['outputs'][0]['files'][0]:
content_length = len(resp_data['outputs'][0]['files'][0]['content'])
f.write(f'Content length: {content_length}\\n')
# Process the response - check for outputs and files
if 'outputs' in resp_data and resp_data['outputs']:
output = resp_data['outputs'][0]
print(f'Found {len(resp_data[\"outputs\"])} outputs in response')
if 'files' in output and output['files']:
file_data = output['files'][0]
print(f'Found {len(output[\"files\"])} files in output')
if 'content' in file_data and file_data['content']:
print(f'Found content in file (length: {len(file_data[\"content\"])})')
with open('$API_DIR/output.$OUTPUT_FORMAT', 'w') as f:
f.write(file_data['content'])
print('CONVERSION SUCCESS')
sys.exit(0)
else:
if 'content' in file_data:
print('Content field exists but is empty')
else:
print('No content field in file data')
print(f'Available fields: {list(file_data.keys())}')
else:
print('No files found in output')
print(f'Available fields: {list(output.keys())}')
# Alternative response format check - document field
elif 'document' in resp_data and resp_data['status'] == 'success':
print('Found alternative response format with document field')
document = resp_data['document'] or {}
# Check format fields in document to see what's available
available_formats = []
if 'html_content' in document and document['html_content']:
available_formats.append(('html', document['html_content']))
if 'md_content' in document and document['md_content']:
available_formats.append(('md', document['md_content']))
if 'text_content' in document and document['text_content']:
available_formats.append(('text', document['text_content']))
if 'json_content' in document and document['json_content']:
available_formats.append(('json', document['json_content']))
if available_formats:
print(f'Found {len(available_formats)} available formats: {[f[0] for f in available_formats]}')
# First try to find the exact requested format
requested_format_match = next((f for f in available_formats if f[0] == '$OUTPUT_FORMAT'.lower()), None)
if requested_format_match:
format_type, content = requested_format_match
print(f'Found content in requested format {format_type} (length: {len(content)})')
else:
# If requested format not found, use the first available
format_type, content = available_formats[0]
print(f'Requested format not found, using alternative format {format_type} (length: {len(content)})')
# Save the content to the output file with appropriate extension
with open(f'$API_DIR/output.{format_type}', 'w') as f:
f.write(content)
# If we're using a different format than requested, also save with requested extension
if format_type != '$OUTPUT_FORMAT'.lower():
print(f'Saving content with requested extension {format_type} -> $OUTPUT_FORMAT')
with open('$API_DIR/output.$OUTPUT_FORMAT', 'w') as f:
f.write(content)
print('CONVERSION SUCCESS')
sys.exit(0)
else:
# No content fields found or all are empty
# Check if fields exist but are empty or null
empty_fields = []
if 'html_content' in document and not document['html_content']:
empty_fields.append('html_content')
if 'md_content' in document and not document['md_content']:
empty_fields.append('md_content')
if 'text_content' in document and not document['text_content']:
empty_fields.append('text_content')
if empty_fields:
print(f'Found content fields but they are empty or null: {empty_fields}')
else:
print('No content fields found in document')
print(f'Available fields in document: {list(document.keys() if document else [])}')
else:
print('No outputs found in response')
print(f'Available fields: {list(resp_data.keys())}')
# Try to extract any alternate formats or metadata
if 'metadata' in resp_data:
print('Metadata found in response, saving to file')
with open('$API_DIR/metadata.json', 'w') as f:
json.dump(resp_data['metadata'], f, indent=2)
print('CONVERSION PARTIAL - Some data available but not complete')
except Exception as json_error:
print(f'Failed to parse response as JSON: {json_error}')
traceback.print_exc()
# Save raw content as text if JSON parsing fails
with open('$API_DIR/output.txt', 'w') as f:
f.write(response.text)
print('Saved raw response as text file')
print('CONVERSION PARTIAL - Raw response saved')
else:
print(f'Error response: {response.text[:500]}')
print('CONVERSION FAILED')
except ImportError:
# Fall back to urllib
import urllib.request
import urllib.error
print('Using urllib library for API call')
headers = {'Content-Type': 'application/json'}
req_data = json.dumps(request_data).encode('utf-8')
req = urllib.request.Request(
'$DOCLING_API_ENDPOINT',
data=req_data,
headers=headers,
method='POST'
)
try:
start_time = time.time()
print(f'Starting conversion request at {time.strftime(\"%H:%M:%S\")}')
with urllib.request.urlopen(req, timeout=300) as response:
elapsed = time.time() - start_time
print(f'Conversion request completed in {elapsed:.2f} seconds')
print(f'Response status: {response.status}')
if response.status == 200:
response_text = response.read().decode('utf-8')
# Save full response for debugging
with open('$API_DIR/full_response.txt', 'w') as f:
f.write(f'Status: {response.status}\\n')
f.write(f'Headers: {response.headers}\\n\\n')
f.write(f'Content: {response_text[:10000]}...' if len(response_text) > 10000 else f'Content: {response_text}')
with open('$API_DIR/response.json', 'w') as f:
f.write(response_text)
try:
resp_data = json.loads(response_text)
print('Successfully parsed response as JSON')
# Save detailed diagnostics about the response structure
with open('$API_DIR/response_structure.txt', 'w') as f:
f.write(f'Response keys: {list(resp_data.keys())}\\n')
if 'document' in resp_data:
f.write(f'Document keys: {list(resp_data[\"document\"].keys() if resp_data[\"document\"] else [])}\\n')
# Check for specific content fields with null safety
doc = resp_data['document'] or {}
if 'html_content' in doc and doc['html_content']:
f.write(f'HTML content length: {len(doc[\"html_content\"])}\\n')
elif 'html_content' in doc:
f.write('HTML content is present but empty or null\\n')
if 'md_content' in doc and doc['md_content']:
f.write(f'Markdown content length: {len(doc[\"md_content\"])}\\n')
elif 'md_content' in doc:
f.write('Markdown content is present but empty or null\\n')
if 'text_content' in doc and doc['text_content']:
f.write(f'Text content length: {len(doc[\"text_content\"])}\\n')
elif 'text_content' in doc:
f.write('Text content is present but empty or null\\n')
if 'json_content' in doc and doc['json_content']:
f.write(f'JSON content length: {len(doc[\"json_content\"])}\\n')
elif 'json_content' in doc:
f.write('JSON content is present but empty or null\\n')
if 'outputs' in resp_data:
f.write(f'Outputs count: {len(resp_data[\"outputs\"])}\\n')
if resp_data['outputs']:
f.write(f'First output keys: {list(resp_data[\"outputs\"][0].keys())}\\n')
if 'files' in resp_data['outputs'][0]:
f.write(f'Files count: {len(resp_data[\"outputs\"][0][\"files\"])}\\n')
if resp_data['outputs'][0]['files']:
f.write(f'First file keys: {list(resp_data[\"outputs\"][0][\"files\"][0].keys())}\\n')
if 'content' in resp_data['outputs'][0]['files'][0]:
content_length = len(resp_data['outputs'][0]['files'][0]['content'])
f.write(f'Content length: {content_length}\\n')
if 'outputs' in resp_data and resp_data['outputs']:
output = resp_data['outputs'][0]
print(f'Found {len(resp_data[\"outputs\"])} outputs in response')
if 'files' in output and output['files']:
file_data = output['files'][0]
print(f'Found {len(output[\"files\"])} files in output')
if 'content' in file_data and file_data['content']:
print(f'Found content in file (length: {len(file_data[\"content\"])})')
with open('$API_DIR/output.$OUTPUT_FORMAT', 'w') as f:
f.write(file_data['content'])
print('CONVERSION SUCCESS')
sys.exit(0)
else:
if 'content' in file_data:
print('Content field exists but is empty')
else:
print('No content field in file data')
print(f'Available fields: {list(file_data.keys())}')
else:
print('No files found in output')
print(f'Available fields: {list(output.keys())}')
# Alternative response format check - document field
elif 'document' in resp_data and resp_data['status'] == 'success':
print('Found alternative response format with document field')
document = resp_data['document'] or {}
# Check format fields in document to see what's available
available_formats = []
if 'html_content' in document and document['html_content']:
available_formats.append(('html', document['html_content']))
if 'md_content' in document and document['md_content']:
available_formats.append(('md', document['md_content']))
if 'text_content' in document and document['text_content']:
available_formats.append(('text', document['text_content']))
if 'json_content' in document and document['json_content']:
available_formats.append(('json', document['json_content']))
if available_formats:
print(f'Found {len(available_formats)} available formats: {[f[0] for f in available_formats]}')
# First try to find the exact requested format
requested_format_match = next((f for f in available_formats if f[0] == '$OUTPUT_FORMAT'.lower()), None)
if requested_format_match:
format_type, content = requested_format_match
print(f'Found content in requested format {format_type} (length: {len(content)})')
else:
# If requested format not found, use the first available
format_type, content = available_formats[0]
print(f'Requested format not found, using alternative format {format_type} (length: {len(content)})')
# Save the content to the output file with appropriate extension
with open(f'$API_DIR/output.{format_type}', 'w') as f:
f.write(content)
# If we're using a different format than requested, also save with requested extension
if format_type != '$OUTPUT_FORMAT'.lower():
print(f'Saving content with requested extension {format_type} -> $OUTPUT_FORMAT')
with open('$API_DIR/output.$OUTPUT_FORMAT', 'w') as f:
f.write(content)
print('CONVERSION SUCCESS')
sys.exit(0)
else:
# No content fields found or all are empty
# Check if fields exist but are empty or null
empty_fields = []
if 'html_content' in document and not document['html_content']:
empty_fields.append('html_content')
if 'md_content' in document and not document['md_content']:
empty_fields.append('md_content')
if 'text_content' in document and not document['text_content']:
empty_fields.append('text_content')
if empty_fields:
print(f'Found content fields but they are empty or null: {empty_fields}')
else:
print('No content fields found in document')
print(f'Available fields in document: {list(document.keys() if document else [])}')
else:
print('No outputs found in response')
print(f'Available fields: {list(resp_data.keys())}')
print('CONVERSION PARTIAL - Some data available but not complete')
except Exception as json_error:
print(f'Failed to parse response as JSON: {json_error}')
traceback.print_exc()
# Save raw content as text if JSON parsing fails
with open('$API_DIR/output.txt', 'w') as f:
f.write(response_text)
print('Saved raw response as text file')
print('CONVERSION PARTIAL - Raw response saved')
else:
print(f'Error status: {response.status}')
print('CONVERSION FAILED')
except urllib.error.HTTPError as e:
print(f'HTTP Error: {e.code} - {e.reason}')
print(f'Response body: {e.read().decode(\"utf-8\")[:500]}')
print('CONVERSION FAILED')
except urllib.error.URLError as e:
print(f'URL Error: {e.reason}')
print('CONVERSION FAILED')
except Exception as e:
print(f'Unexpected error during urllib request: {e}')
traceback.print_exc()
print('CONVERSION FAILED')
except Exception as e:
print(f'Error during conversion: {e}')
traceback.print_exc()
print('CONVERSION FAILED')
" 2>&1
# --- Check for various potential output files ---
echo "Checking for output files..."
if [ -f "$API_DIR/output.$OUTPUT_FORMAT" ]; then
echo "Conversion completed successfully! Output file found."
# Get content from the converted file
OUTPUT_SIZE=$(wc -c < "$API_DIR/output.$OUTPUT_FORMAT")
echo "Output file found with size: $OUTPUT_SIZE bytes"
# Calculate the access URL for result display
RESULT_URL="https://api.apify.com/v2/key-value-stores/${APIFY_DEFAULT_KEY_VALUE_STORE_ID}/records/OUTPUT"
echo "=============================="
echo "PROCESSING COMPLETE!"
echo "Document URL: ${DOCUMENT_URL}"
echo "Output format: ${OUTPUT_FORMAT}"
echo "Output size: ${OUTPUT_SIZE} bytes"
echo "=============================="
# Set the output content type based on format
CONTENT_TYPE="text/plain"
case "$OUTPUT_FORMAT" in
md) CONTENT_TYPE="text/markdown" ;;
html) CONTENT_TYPE="text/html" ;;
json) CONTENT_TYPE="application/json" ;;
text) CONTENT_TYPE="text/plain" ;;
esac
# Upload the document content using our function
upload_to_kvs "$API_DIR/output.$OUTPUT_FORMAT" "OUTPUT" "$CONTENT_TYPE" "Document content"
# Only proceed with dataset record if document upload succeeded
if [ $? -eq 0 ]; then
echo "Your document is available at: ${RESULT_URL}"
echo "=============================="
# Find the Apify CLI again (reusing the function's logic would be better, but for clarity we'll repeat)
APIFY_CMD=""
for cmd in "apify" "actor" "/usr/local/bin/apify" "/usr/bin/apify" "/opt/apify/cli/bin/apify"; do
if command -v "$cmd" &> /dev/null; then
APIFY_CMD="$cmd"
break
fi
done
if [ -n "$APIFY_CMD" ]; then
# Add record to dataset with enhanced version check prevention
echo "Adding record to dataset..."
# Create a temporary home directory with write permissions
export TMPDIR="/tmp/apify-home-${RANDOM}"
mkdir -p "$TMPDIR"
# Multiple strategies to disable version checking
export APIFY_DISABLE_VERSION_CHECK=1
export NODE_OPTIONS="--no-warnings"
export HOME="$TMPDIR" # Override home directory to writable location
# Use the --no-update-notifier flag if available
if $APIFY_CMD --help | grep -q "\--no-update-notifier"; then
if $APIFY_CMD --no-update-notifier actor:push-data "{\"url\": \"${DOCUMENT_URL}\", \"output_file\": \"${RESULT_URL}\", \"status\": \"success\"}"; then
echo "Successfully added record to dataset"
else
echo "Warning: Failed to add record to dataset"
fi
else
# Fall back to regular command
if $APIFY_CMD actor:push-data "{\"url\": \"${DOCUMENT_URL}\", \"output_file\": \"${RESULT_URL}\", \"status\": \"success\"}"; then
echo "Successfully added record to dataset"
else
echo "Warning: Failed to add record to dataset"
fi
fi
rm -rf "$TMPDIR" 2>/dev/null || true # Clean up temp dir
fi
fi
else
echo "ERROR: No converted output file found at $API_DIR/output.$OUTPUT_FORMAT"
# Create error metadata
ERROR_METADATA="{\"status\":\"error\",\"error\":\"No converted output file found\",\"documentUrl\":\"$DOCUMENT_URL\"}"
echo "$ERROR_METADATA" > "/tmp/actor-output/OUTPUT"
chmod 644 "/tmp/actor-output/OUTPUT"
echo "Error information has been saved to /tmp/actor-output/OUTPUT"
fi
# --- Verify output files for debugging ---
echo "=== Final Output Verification ==="
echo "Files in /tmp/actor-output:"
ls -la /tmp/actor-output/ 2>/dev/null || echo "Cannot list /tmp/actor-output/"
echo "All operations completed. The output should be available in the default key-value store."
echo "Content URL: ${RESULT_URL:-No URL available}"
# --- Cleanup function ---
cleanup() {
echo "Running cleanup..."
# Stop the API process
if [ -n "$API_PID" ]; then
echo "Stopping docling-serve API (PID: $API_PID)..."
kill $API_PID 2>/dev/null || true
fi
# Export log file to KVS if it exists
# DO THIS BEFORE REMOVING TOOLS DIRECTORY
if [ -f "$LOG_FILE" ]; then
if [ -s "$LOG_FILE" ]; then
echo "Log file is not empty, pushing to key-value store (key: LOG)..."
# Upload log using our function
upload_to_kvs "$LOG_FILE" "LOG" "text/plain" "Log file"
else
echo "Warning: log file exists but is empty"
fi
else
echo "Warning: No log file found"
fi
# Clean up temporary files AFTER log is uploaded
echo "Cleaning up temporary files..."
if [ -d "$API_DIR" ]; then
echo "Removing API working directory: $API_DIR"
rm -rf "$API_DIR" 2>/dev/null || echo "Warning: Failed to remove $API_DIR"
fi
if [ -d "$TOOLS_DIR" ]; then
echo "Removing tools directory: $TOOLS_DIR"
rm -rf "$TOOLS_DIR" 2>/dev/null || echo "Warning: Failed to remove $TOOLS_DIR"
fi
# Keep log file until the very end
echo "Script execution completed at $(date)"
echo "Actor execution completed"
}
# Register cleanup
trap cleanup EXIT
Reference in New Issue View Git Blame Copy Permalink