9.1

Edit

Work with S3 bucket

Save content on S3

The following config saves the specified fields content on S3 (useful for dealing with *_tagged fields)

  • Overrides the fields content by S3 link
  • Skips the fields which are not present in the input
  • Skips the fields which are already an http link (safe to use the step several times in the flow)
  • Adds proper UTF-8 headers

Save Content on S3

<?xml version="1.0" encoding="UTF-8"?>
<config charset="UTF-8">
  
    <var-def name="variablesToSave">
        <loop item="varName">
            <list>
                <tokenize delimiters="|">
                    document_xml_link_tagged|document_xml_link_tagged_failed
                </tokenize>
            </list>
            <body>
                <case>
                    <if condition='${sys.isVariableDefined(varName.toString())}'>
                        <var name="varName"/>
                    </if>
                </case>
            </body>
        </loop>
    </var-def>
  
    <loop item="taggedVar">
        <list>
            <var name="variablesToSave"/>
        </list>
        <body>
            <var-def name="content">
                <var name="${taggedVar}"/>
            </var-def>
      
            <var-def name="${taggedVar}">
                <case>
                    <if condition='${content.toString().startsWith("http")}'>
                        <var name="content"/>
                    </if>
                    <else>
                        <s3 bucket="${conf_invoice_image_bucket}">
                            <s3-put path="tagged/${java.util.UUID.randomUUID()}-tagged.html" acl="PublicRead"
                                    content-type="text/html; charset=utf-8" content-disposition="inline">
                                <var name='content'/>
                            </s3-put>
                        </s3>
                    </else>
                </case>
            </var-def>
        </body>
    </loop>
    <export include-original-data="true">
        <loop item="taggedVar">
            <list>
                <var name="variablesToSave"/>
            </list>
            <body>
                <single-column name="${taggedVar}">
                    <var name="${taggedVar}"/>
                </single-column>
            </body>
        </loop>
    </export>
</config>

Download ZIP from S3 and extract content

This is a universal example – an archive can contain files and nested folders.

Read archive from S3

<?xml version="1.0" encoding="UTF-8"?>
<config xmlns="http://web-harvest.sourceforge.net/schema/1.0/config" charset="UTF-8">

    <s3 bucket="your-bucket">
        <var-def name="sourceFile">
            <unzip>
                <s3-get name="testfolder/test_arch.zip"/>
            </unzip>
        </var-def>
    </s3>
    <script><![CDATA[
        import com.freedomoss.crowdcontrol.webharvest.plugin.zip.dto.FileEntity;
        import com.freedomoss.crowdcontrol.webharvest.plugin.zip.dto.FileType; // used only to print file type

        // <unzip> returns ListVariable with single (or empty) NodeVariable inside which wrap FileEntity
        FileEntity root = sourceFile.getWrappedObject().get(0).getWrappedObject();
        // This root FileEntity is always directory >>> root.isDirectory() always return true
        List files = root.getChildren();

        void processFilesInDirectory(FileEntity dir) {
            if (dir.isDirectory()) {
                for (FileEntity fileEntity : files) {
                    if (fileEntity.isFile()) {
                        processFile(fileEntity);
                    } else {
                        // can be recursive call
                        processFilesInDirectory(fileEntity);
                    }
                }
            }
        }

        void processFile(FileEntity fileEntity) {
            // do something with content
            // ...

            // For example, log content and all available data from FileEntity

            byte[] content = fileEntity.getContent();
            String fileName = fileEntity.getName();
            FileType fileType = fileEntity.getType(); // always return FileType.FILE for files
            String mimeType = fileEntity.getMimeType();
            List children = fileEntity.getChildren(); // always return null for files

            String toString = fileEntity.toString();

            log.warn("File name: {}", fileName);
            log.warn("File type: {}", fileType);
            log.warn("File mime type: {}", mimeType);
            log.warn("File children: {}", children == null ? "null" : children.toString());
            log.warn("File content size: {}", content.length);
        }

        processFilesInDirectory(root);
    ]]></script>


    <export include-original-data="true"></export>
</config>

Save Data Store on S3 in CSV format

Provide bucket name and Data Store name for the snippet below:

Saving Data Store on S3

    <var-def name="outputLink">
        <s3 bucket="${bucketName}">
            <s3-put path="datastore/${datastoreName}.csv" acl="PublicRead"
                    content-type="text/csv; charset=utf-8" content-disposition="inline">
                <list-to-csv>
                    <json expression="$.row">
                        <xml-to-json>
                            <datastore name="${datastoreName}">
                                select * from @this;
                            </datastore>
                        </xml-to-json>
                    </json>
                </list-to-csv>
            </s3-put>
        </s3>
    </var-def>

← PreviousNext →