Edit

Convert files

TIFF to PDF

tiff2pdf needs to be installed on the linux environment, where WorkFusion is running. tiff2pdf is not a part of the WorkFusion offering.

Major browsers have some limitations displaying TIFF files inside (most of them propose to download original tiff). But sometimes there is a need to show content of tiff files in Human Tasks (for example, for data extraction).

Widely used solution is to convert original TIFF file into a PDF and show the converted PDF inside your Manual Task.

We recommend to use a built-in Linux library – tiff2pdf (it's not part of WorkFusion package). Below you can find example of Bot config, which converts a TIFF file into a PDF and uploads the converted PDF to S3 file storage.

TIFF to PDF

<var-def name="convertedPDFLink">
    <script return="convert(document_link.toString())"><![CDATA[
        import java.io.BufferedReader;
        import java.io.File;
        import java.io.IOException;
        import java.io.InputStreamReader;
        import java.net.URL;
        import com.itextpdf.text.Image;

        import com.google.common.io.Files;
        import org.apache.commons.io.FileUtils;
        import org.apache.commons.io.FilenameUtils;
        import com.freedomoss.workfusion.utils.gson.GsonUtils;

        static File convert(String documentLink) throws IOException, InterruptedException {
            File baseDir = Files.createTempDir();
            
            File inputFolder = new File(baseDir, "input");

            URL documentUrl = new URL(documentLink);
            String inputFileName = FilenameUtils.getName(documentUrl.getPath());
            if (inputFileName == null || inputFileName.isEmpty()) {
                inputFileName = "input.tiff";
            }

            File inputFile = new File(inputFolder, inputFileName);
            FileUtils.copyURLToFile(documentUrl, inputFile);
            
            File outputFolder = new File(baseDir, "output");
            FileUtils.forceMkdir(outputFolder);

            File result = new File(outputFolder, "output.pdf");                  

            invoke(new ProcessBuilder(new String[] {
                "tiff2pdf", 
                inputFile.getAbsolutePath(),
                "-o",                             
                result.getAbsolutePath()
                })); 
            FileUtils.forceDelete(inputFolder);
            return result;
        }

        static void invoke(ProcessBuilder builder) throws IOException, InterruptedException {
            builder.redirectErrorStream(true);

            System.out.println(builder.command());
            Process process = builder.start();

            BufferedReader in = new BufferedReader(new InputStreamReader(process.getInputStream()));
            String line;
            while ((line = in.readLine()) != null) {
                System.out.println(line);
            }
            in.close();

            int code = process.waitFor();
            if (code != 0) {
                throw new RuntimeException("Failed to invoke process: " + builder.command() + ". Return code: " + code);
            }
        }

    ]]></script>
</var-def>

<var-def name="content">
    <file path="${convertedPDFLink}" type="binary"/>
</var-def>

<var-def name="converted_pdf_link">
    <s3  bucket="temp_bucket">
      <s3-put path="conver/${document_uuid}.pdf" content="${content}" content-type="application/pdf" content-disposition="inline"/>
    </s3>
</var-def>

Another version of the TIFF to PDF conversion, where user actually has control over quality of the output PDF file, is to use ImageMagic with GhostScript. If both utilities are installed on the Linux where WF is running, you may try changing invoke call to something like:

invoke(new ProcessBuilder(new String[] {
                "convert",
                "-limit","memory", "0",
                "-limit", "map", "0",
                inputFile.getAbsolutePath(),
        "-compress", "jpeg", 
            "-quality", "40",                        
                result.getAbsolutePath()}
              ));

The compress and quality parameters for convert utility are worth experimenting with to produce PDF of the acceptable quality and size.

PDF to Image

You may convert PDF document to a set of images. For example, if you need to pass those images to OCR. Refer javadoc for details.

<?xml version="1.0" encoding="UTF-8"?>
<config xmlns="http://web-harvest.sourceforge.net/schema/1.0/config" scriptlang="groovy">
    <script><![CDATA[
        //Supported formats: JPG jpg bmp BMP gif GIF WBMP png PNG wbmp jpeg JPEG
        String[] a = ["-imageType", "png", "-outputPrefix", "/path/to/output_folder/output-", "/path/to/input.pdf"];
        org.apache.pdfbox.PDFToImage.main(a);

    ]]></script>
    <export include-original-data="true"></export>
</config>

PDF to XML

On many online resources one may find that the task of conversion of PDF file into XML representation is like converting hamburgers to cows.

In any case, here is a few options to do that (using Apache PDFBox and using Apache Tika):

PDF to XML

<?xml version="1.0" encoding="UTF-8"?>
<config xmlns="http://web-harvest.sourceforge.net/schema/1.0/config" scriptlang="groovy">

    <script><![CDATA[
        import org.apache.pdfbox.pdmodel.PDDocument;
        import org.apache.pdfbox.util.PDFText2HTML;
        
        import org.apache.tika.metadata.Metadata;
        import org.apache.tika.parser.ParseContext;
        import org.apache.tika.parser.Parser;
        import org.apache.tika.parser.pdf.PDFParser;
        import org.apache.tika.sax.ToXMLContentHandler;

        class PdfConverter {

            public String pdfToHtml(byte[] content) {
                PDDocument pddDocument = PDDocument.load(new ByteArrayInputStream(content));
                PDFText2HTML stripper = new PDFText2HTML("UTF-8");
                return stripper.getText(pddDocument);
            }

            public String pdfToXmlTika(byte[] content) {
                InputStream input = new ByteArrayInputStream(content);
                Metadata metadata = new Metadata();
                ParseContext context = new ParseContext();
                Parser parser = new PDFParser();
        
                ToXMLContentHandler handler = new ToXMLContentHandler();
                parser.parse(input, handler, metadata, context);
        
                return handler.toString();
            }
        }
        0;
        
    ]]></script>

    <var-def name="pdfContent">
        <file path="xyz/file.pdf" type="binary"/>
    </var-def>

    <var-def name="xmlUsingTika">
        <script return="new PdfConverter().pdfToXmlTika(pdfContent.toBinary())"/>
    </var-def>

    <var-def name="xmlUsingPdfBox">
        <html-to-xml omitdoctype="true">
            <script return="new PdfConverter().pdfToHtml(pdfContent.toBinary())"/>
        </html-to-xml>
    </var-def>

    <export include-original-data="true"></export>

</config>

PDF Manipulations: cut, get number of pages

PDF manipulation functions

<?xml version="1.0" encoding="UTF-8"?>
<config charset="UTF-8">
  
  <function name="getCombinedDocumentsLink">
    <return>
        <empty>
            <var-def name="combinedPath">
              <script return="getCombinedDocumentsAbsolutePath(document_link.toString())">
                <![CDATA[ 
                
                    import java.io.BufferedReader;
                    import java.io.File;
                    import java.io.IOException;
                    import java.io.InputStreamReader;
                    import java.net.URL;
                    
                    import com.google.common.io.Files;
                    import org.apache.commons.io.FileUtils;
                    import org.apache.commons.io.FilenameUtils;
                    import org.apache.commons.lang3.ArrayUtils;

                    File baseDir = null;
              
                    public String getCombinedDocumentsAbsolutePath(String documentLink) throws IOException, InterruptedException {
              
                        baseDir = Files.createTempDir();
                        //System.out.println(baseDir);
                
                        File inputFolder = new File(baseDir, "input");
              
                        String[] links =  documentLink.split("\\|");
                        String[] inputFilesStr= new String[links.length];
                        for(int i=0;i<links.length;i++){
              
                            //Download documents to file system
                            URL documentUrl = new URL(links[i]);
                            String inputFileName = FilenameUtils.getName(documentUrl.getPath());
                            if (inputFileName == null || inputFileName.isEmpty()) {
                                inputFileName = "input"+i+".pdf";
                            }
                    
                            File inputFile = new File(inputFolder, inputFileName);
                            FileUtils.copyURLToFile(documentUrl, inputFile);
                            String fileName=inputFile.getAbsolutePath();
              
                            if (fileName.contains(".tif")) {
                              File inputFilePdf = new File(inputFolder, inputFileName + ".pdf");
                              fileName=inputFilePdf.getAbsolutePath();
                              invoke(new ProcessBuilder(new String[] {
                                "convert",
                                inputFile.getAbsolutePath(),
                                inputFilePdf.getAbsolutePath()}));
                            }
                            inputFilesStr[i]=fileName;
                        }
                
                        File outputFolder = new File(baseDir, "output");
                        FileUtils.forceMkdir(outputFolder);
              
                        File result = new File(baseDir +"/output", "res.pdf");
              
                        String[] gsParams=new String[] {
                                "gs",
                                "-q",
                                "-dBATCH",
                                "-dNOPAUSE",
                                "-sDEVICE=pdfwrite",
                                "-sOutputFile=" + outputFolder.getAbsolutePath() + "/res.pdf"};
              
                        String[] invokeParams=ArrayUtils.addAll(gsParams, inputFilesStr);
                        String[] invokeParams=ArrayUtils.addAll(invokeParams, new String[]{"-c", "quit"});
                
                        invoke(new ProcessBuilder(invokeParams));
                
                        return result.getAbsolutePath();
                    }
                
                    public void invoke(ProcessBuilder builder) throws IOException, InterruptedException {
                        builder.redirectErrorStream(true);
              
                        System.out.println(builder.command());
                        Process process = builder.start();
                
                        BufferedReader in = new BufferedReader(new InputStreamReader(process.getInputStream()));
                        String line;
                        while ((line = in.readLine()) != null) {
                            System.out.println(line);
                        }
                        in.close();
                
                        int code = process.waitFor();
                        if (code != 0) {
                            throw new RuntimeException("Failed to invoke process: " + builder.command() + ". Return code: " + code);
                        }
                    }
              ]]></script>
            </var-def>
    
          
            <var-def name="content">
                <file path="${combinedPath}" type="binary"/>
            </var-def>
        </empty>
  
  
        <var-def name="combined_file_link">
            <s3  bucket="${bucket.toString()}">
                <s3-put-public path='${targetFolder.toString()}/${targetFileName.toString()}.pdf' content="${content}" content-type="application/pdf" content-disposition="inline"/>
            </s3>
        </var-def>
      
        <empty>
          <script><![CDATA[ 
                if(baseDir != null){
                    FileUtils.deleteDirectory(baseDir);
                 }
          ]]></script>
        </empty>
      </return>
  </function>
  
  
  <function name="calculateNumberOfPages">
    <return>
      <var-def name="pageCount">
        <script return="getPageCount(document_link.toString())">
          <![CDATA[ 
          
              import java.io.IOException;

              import com.lowagie.text.pdf.PdfReader;
              import com.lowagie.text.pdf.RandomAccessFileOrArray;
 
        
              public int getPageCount(String documentLink) throws IOException {
                  RandomAccessFileOrArray file = new RandomAccessFileOrArray(documentLink, false, true );
                  PdfReader reader = new PdfReader(file, null);
                  int ret = reader.getNumberOfPages();
                  reader.close();
                  return ret; 
              }
             
        ]]></script>
      </var-def>
    </return>
  </function>
  
  
  <function name="cutPagesFromPdf">
    <return>
      <empty>
        
          <var-def name="resultLink">
            <var name="document_link"/>
          </var-def>
        
         <var-def name="pagesCount">
            <call name="calculateNumberOfPages">
                 <call-param name="document_link">
                    <var name="document_link"/>
                 </call-param>
            </call>
         </var-def>
        
        <script><![CDATA[ 
                Integer firstXPages = Integer.valueOf(firstPagesCount.toString());
                Integer lastXPages = Integer.valueOf(lastPagesCount.toString());
                Integer totalPagesCount = Integer.valueOf(pagesCount.toString());
        ]]></script>
        
         <case>
            <if condition="${totalPagesCount > firstXPages + lastXPages}">
              <script>
                  <![CDATA[ 
                      import java.io.ByteArrayOutputStream;
                      import java.net.URL;
                      
                      import com.itextpdf.text.pdf.PdfReader;
                      import com.itextpdf.text.pdf.PdfStamper;
                      import org.apache.commons.lang3.StringUtils;

          
                      PdfReader document = new PdfReader(new URL(document_link.toString()));
                      List pdfPages = new ArrayList();
      
                      for(int i = 1; i <= firstXPages; i++){
                        pdfPages.add(i);
                      }
                      for(int i = lastXPages; i > 0 ; i--){
                        pdfPages.add(totalPagesCount - i + 1);
                      }
              
                      document.selectPages(StringUtils.join(pdfPages, ","));
              
                      ByteArrayOutputStream cutPdfOutput = new ByteArrayOutputStream();
              
                      PdfStamper pdfStamper = new PdfStamper(document, cutPdfOutput);
                      pdfStamper.close();
              
                      document.close();
      
                      int lastIndexOfSlash=targetFileName.toString().lastIndexOf("/");
                      String newFileS3Name = targetFileName.toString().substring(lastIndexOfSlash).replaceAll("[^A-Za-z0-9]", "_") + "-cut.pdf";
                   ]]>
              
              </script>
            
              <var-def name="content">
                  <script return="cutPdfOutput.toByteArray()"/>
              </var-def>
            
              <var-def name="resultLink">
                  <s3 bucket="${bucket.toString()}">
                      <s3-put-public path='${targetFolder.toString()}/${newFileS3Name}' content="${content}" content-type="application/pdf" content-disposition="inline"/>
                  </s3>
              </var-def>
            </if>
        </case>
      </empty>
            
      <var-def name="result">
        <var name="resultLink"/>
      </var-def>
      
    </return>
  </function>
  
  
  <function name="getCombinedDocumentsWithCuttingLongFilesLink">
    <return>
      <empty>
        <script><![CDATA[ 
                String[] s3links =  document_link.toString().split("\\|");
                List resultLinks = new ArrayList(s3links.length);
        ]]></script>
        
        <loop item="s3link" index="loopIndex" >
          <list>
              <script return="s3links"/>
          </list>
          <body>
               <var-def name="resultLink">
                 <call name="cutPagesFromPdf">
                    <call-param name="document_link">
                        <var name="s3link"/>
                    </call-param>
                    <call-param name="firstPagesCount">
                        <var name="firstPagesCount"/>
                    </call-param>
                    <call-param name="lastPagesCount">
                        <var name="lastPagesCount"/>
                    </call-param>
                    <call-param name="bucket">
                        <var name="bucket"/>
                    </call-param>
                    <call-param name="targetFolder">
                        <var name="targetFolder"/>
                    </call-param>
                    <call-param name="targetFileName">
                        <var name="s3link"/>
                    </call-param>
                </call>
              </var-def>

              <script><![CDATA[ 
                  resultLinks.add(resultLink.toString());
              ]]></script>
          </body>
        </loop>
        
        <var-def name="document_link_cutted">
            <template>${org.apache.commons.lang3.StringUtils.join(resultLinks, "|")}</template>
        </var-def>
      </empty>
            
      <var-def name="combined_file_link">
        <call name="getCombinedDocumentsLink">
            <call-param name="document_link">
                <var name="document_link_cutted"/>
            </call-param>
            <call-param name="bucket">
                <var name="bucket"/>
            </call-param>
            <call-param name="targetFolder">
                <var name="targetFolder"/>
            </call-param>
            <call-param name="targetFileName">
                <var name="document_uuid"/>
            </call-param>
        </call>
      </var-def>  
    </return>
  </function>
</config>

These functions can be used with the following parameters:

Function usage

<!-- getCombinedDocumentsLink - combine pdf/tiff files into one pdf file (document_link is | separated links) -->
<var-def name="combined_file_link">
    <call name="getCombinedDocumentsLink">
        <call-param name="document_link">
            <var name="document_link"/>
        </call-param>
        <call-param name="bucket">
            <template>stretiakov</template>
        </call-param>
        <call-param name="targetFolder">
            <template>join</template>
        </call-param>
        <call-param name="targetFileName">
            <var name="document_uuid"/>
        </call-param>
    </call>
</var-def>
 
 
<!-- calculateNumberOfPages - return pages count in pdf file. -->
<var-def name="pagesCount">
    <call name="calculateNumberOfPages">
         <call-param name="document_link">
            <var name="s3link"/>
         </call-param>
    </call>
</var-def>
 
 
<!-- cutPagesFromPdf - create pdf with 3 first and 2 last pages from document and load it to s3. -->
<var-def name="resultLink">
   <call name="cutPagesFromPdf">
      <call-param name="document_link">
          <var name="s3link"/>
      </call-param>
      <call-param name="firstPagesCount">
          <var name="firstPagesCount"/>
      </call-param>
      <call-param name="lastPagesCount">
          <var name="lastPagesCount"/>
      </call-param>
      <call-param name="bucket">
          <var name="bucket"/>
      </call-param>
      <call-param name="targetFolder">
          <var name="targetFolder"/>
      </call-param>
      <call-param name="targetFileName">
          <var name="s3link"/>
      </call-param>
  </call>
</var-def>
 
 
<!-- getCombinedDocumentsWithCuttingLongFilesLink - combine pdfs into one document. If pdf has more then 5 pages then cut this document. -->
<var-def name="combined_file_link">
  <call name="getCombinedDocumentsWithCuttingLongFilesLink">
      <call-param name="document_link">
          <var name="document_link"/>
      </call-param>
      <call-param name="bucket">stretiakov</call-param>
      <call-param name="targetFolder">join</call-param>
      <call-param name="firstPagesCount">1</call-param>
      <call-param name="lastPagesCount">1</call-param>
      <call-param name="targetFileName">
          <var name="document_uuid"/>
      </call-param>
  </call>
</var-def>

How to see tagged text locally the same as in IE answer

Run the following example using WorkFusion Studio. In the Bot config, replace the path to local files: INSERT_YOUR_PATH_TO_FOLDER_WITH_FILES

Converting HTML tagged document

<?xml version="1.0" encoding="UTF-8"?>
<config>
    <var-def name="viewer">
        <![CDATA[ 
            <link href="https://s3.amazonaws.com/public.crowdcontrol/ocr/ocr-xml-viewer-0.3-scale-to-width.css" rel="stylesheet" type="text/css"/>
            <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.8.3/jquery.min.js" type="text/javascript"></script>
            <script src="https://s3.amazonaws.com/public.crowdcontrol/ocr/ocr-xml-viewer-0.3-scale-to-width.js"></script>
    
            <script>
                $(document).ready(function() {
                    generate($('body'), 1.8);
                });
            </script>
         ]]>
    </var-def>

    <loop item="file">
        <list>
            <script return='new File("INSERT_YOUR_PATH_TO_FOLDER_WITH_FILES").listFiles()'/>
        </list>
        <body>
            <var-def name="content">
                <file path="${file}"></file>
            </var-def>

              <var-def name="content">
                <script return="withViewer">
                    <![CDATA[ 
                        String withViewer = content.toString().replaceAll("(<document[^>]*>)", "$1" + java.util.regex.Matcher.quoteReplacement(viewer.toString()));
                     ]]>
                </script>
              </var-def>

              <file path="${file}" action="write">
                <var name="content"/>
              </file>
        </body>
    </loop>
</config>