mirror of
https://github.com/docling-project/docling.git
synced 2025-06-27 05:20:05 +00:00
5458a88464
* add coverage calculation and push Signed-off-by: Michele Dolfi <dol@zurich.ibm.com> * new codecov version and usage of token Signed-off-by: Michele Dolfi <dol@zurich.ibm.com> * enable ruff formatter instead of black and isort Signed-off-by: Michele Dolfi <dol@zurich.ibm.com> * apply ruff lint fixes Signed-off-by: Michele Dolfi <dol@zurich.ibm.com> * apply ruff unsafe fixes Signed-off-by: Michele Dolfi <dol@zurich.ibm.com> * add removed imports Signed-off-by: Michele Dolfi <dol@zurich.ibm.com> * runs 1 on linter issues Signed-off-by: Michele Dolfi <dol@zurich.ibm.com> * finalize linter fixes Signed-off-by: Michele Dolfi <dol@zurich.ibm.com> * Update pyproject.toml Co-authored-by: Cesar Berrospi Ramis <75900930+ceberam@users.noreply.github.com> Signed-off-by: Michele Dolfi <97102151+dolfim-ibm@users.noreply.github.com> --------- Signed-off-by: Michele Dolfi <dol@zurich.ibm.com> Signed-off-by: Michele Dolfi <97102151+dolfim-ibm@users.noreply.github.com> Co-authored-by: Cesar Berrospi Ramis <75900930+ceberam@users.noreply.github.com>
82 lines
2.6 KiB
Python
Vendored
82 lines
2.6 KiB
Python
Vendored
import json
|
|
import logging
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
from docling.backend.pypdfium2_backend import PyPdfiumDocumentBackend
|
|
from docling.datamodel.base_models import InputFormat
|
|
from docling.document_converter import (
|
|
DocumentConverter,
|
|
PdfFormatOption,
|
|
WordFormatOption,
|
|
)
|
|
from docling.pipeline.simple_pipeline import SimplePipeline
|
|
from docling.pipeline.standard_pdf_pipeline import StandardPdfPipeline
|
|
|
|
_log = logging.getLogger(__name__)
|
|
|
|
|
|
def main():
|
|
input_paths = [
|
|
Path("README.md"),
|
|
Path("tests/data/html/wiki_duck.html"),
|
|
Path("tests/data/docx/word_sample.docx"),
|
|
Path("tests/data/docx/lorem_ipsum.docx"),
|
|
Path("tests/data/pptx/powerpoint_sample.pptx"),
|
|
Path("tests/data/2305.03393v1-pg9-img.png"),
|
|
Path("tests/data/pdf/2206.01062.pdf"),
|
|
Path("tests/data/asciidoc/test_01.asciidoc"),
|
|
]
|
|
|
|
## for defaults use:
|
|
# doc_converter = DocumentConverter()
|
|
|
|
## to customize use:
|
|
|
|
doc_converter = (
|
|
DocumentConverter( # all of the below is optional, has internal defaults.
|
|
allowed_formats=[
|
|
InputFormat.PDF,
|
|
InputFormat.IMAGE,
|
|
InputFormat.DOCX,
|
|
InputFormat.HTML,
|
|
InputFormat.PPTX,
|
|
InputFormat.ASCIIDOC,
|
|
InputFormat.CSV,
|
|
InputFormat.MD,
|
|
], # whitelist formats, non-matching files are ignored.
|
|
format_options={
|
|
InputFormat.PDF: PdfFormatOption(
|
|
pipeline_cls=StandardPdfPipeline, backend=PyPdfiumDocumentBackend
|
|
),
|
|
InputFormat.DOCX: WordFormatOption(
|
|
pipeline_cls=SimplePipeline # , backend=MsWordDocumentBackend
|
|
),
|
|
},
|
|
)
|
|
)
|
|
|
|
conv_results = doc_converter.convert_all(input_paths)
|
|
|
|
for res in conv_results:
|
|
out_path = Path("scratch")
|
|
print(
|
|
f"Document {res.input.file.name} converted."
|
|
f"\nSaved markdown output to: {out_path!s}"
|
|
)
|
|
_log.debug(res.document._export_to_indented_text(max_text_len=16))
|
|
# Export Docling document format to markdowndoc:
|
|
with (out_path / f"{res.input.file.stem}.md").open("w") as fp:
|
|
fp.write(res.document.export_to_markdown())
|
|
|
|
with (out_path / f"{res.input.file.stem}.json").open("w") as fp:
|
|
fp.write(json.dumps(res.document.export_to_dict()))
|
|
|
|
with (out_path / f"{res.input.file.stem}.yaml").open("w") as fp:
|
|
fp.write(yaml.safe_dump(res.document.export_to_dict()))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|