Edit

Train model locally

A model generated from an Archetype contains a local runner that allows training the configured model. The runner is located at ../run/ModelTrainingRunner.java.

The ModelTrainingRunner class contains a single method. For Information Extraction, provide a path to the dataset and a working directory and specify a FieldInfo field.

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.workfusion.vds.sdk.api.nlp.configuration.FieldInfo;
import com.workfusion.vds.sdk.api.nlp.configuration.FieldType;
import com.workfusion.vds.sdk.run.ModelRunner;
import com.workfusion.vds.sdk.run.config.LocalTrainingConfiguration;

public class ModelTrainingRunner {

    public static void main(String[] args) throws Exception {
        System.setProperty("WORKFLOW_LOG_FOLDER", "./logs/");

        //Configure input and output
        Path inputDirPath = Paths.get("TODO add path to folder with training set");
        Path outputDirPath = Paths.get("TODO add path to output folder");

        //Configure the fields according to your use case
        List<FieldInfo> fields = new ArrayList<>();
        fields.add(new FieldInfo.Builder("invoice_number")
            .type(FieldType.INVOICE_TYPE)
            .build());
        fields.add(new FieldInfo.Builder("total_amount")
            .type(FieldType.PRICE)
            .build());
        //Add parameters, if needed
        Map<String, Object> parameters = new HashMap<>();

        LocalTrainingConfiguration configuration = LocalTrainingConfiguration.builder()
                .inputDir(inputDirPath)
                .outputDir(outputDirPath)
                .fields(fields)
                .parameters(parameters)
                .build();
        //Run your model specified by the Hypermodel class and configuration
        ModelRunner.run(TrainingIeModel.class, configuration);
    }
}

When training is finished, the resulting model is available at workingDir/training/output/model. The path is passed to the execution runner to check the model quality.

Local model training

Run ModelTrainingRunner

To make the runner work, provide the following parameters:

To make the runner work, provide the following parameters:

• inputDirPath is the path to the folder with training set files.
• outputDirPath is the path to the output folder where trained model artifacts are stored.
• FieldInfo fields represent the configuration of an Information Extraction model (if you use one).
• ModelRunner.run() is your custom model class.
• Optional: add additional parameters, for example, additional training configuration, and so on.
• Optional: add additional inputs, such as dictionaries, reference data, and so on. Use the addResource(targetFolder, sourceFile/Folder) method.

Here is an example of a ModelTrainingRunner configuration for invoice_number single field and a corresponding type:

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.workfusion.vds.sdk.api.nlp.configuration.FieldInfo;
import com.workfusion.vds.sdk.api.nlp.configuration.FieldType;
import com.workfusion.vds.sdk.run.ModelRunner;
import com.workfusion.vds.sdk.run.config.LocalTrainingConfiguration;

public class ModelTrainingRunner {

    public static void main(String[] args) throws Exception {
        System.setProperty("WORKFLOW_LOG_FOLDER", "./logs/");

        Path inputDirPath = Paths.get("/home/ie-example/input");
        Path outputDirPath = Paths.get("/home/ie-example/output");

        List<FieldInfo> fields = new ArrayList<>();
        fields.add(new FieldInfo.Builder("invoice_number")
                .type(FieldType.INVOICE_NUMBER)
                .required(true)
                .multiValue(false)
                .build());

        Map<String, Object> parameters = new HashMap<>();

        LocalTrainingConfiguration configuration = LocalTrainingConfiguration.builder()
                .inputDir(inputDirPath)
                .outputDir(outputDirPath)
                .fields(fields)
                .parameters(parameters)
                .addResource("dictionary", Paths.get("home/ie-example/dictionary/InvoiceNumbers.csv"))
                .build();

        ModelRunner.run(ExampleIeModel.class, configuration);
    }
}

Here is an example of a ModelTrainingRunner configuration for a the invoice_group group field and a corresponding type:

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.workfusion.vds.sdk.api.nlp.configuration.FieldInfo;
import com.workfusion.vds.sdk.api.nlp.configuration.FieldType;
import com.workfusion.vds.sdk.run.ModelRunner;
import com.workfusion.vds.sdk.run.config.LocalTrainingConfiguration;

public class ModelTrainingRunner {

    public static void main(String[] args) throws Exception {
        System.setProperty("WORKFLOW_LOG_FOLDER", "./logs/");

        Path inputDirPath = Paths.get("/home/ie-example/input");
        Path outputDirPath = Paths.get("/home/ie-example/output");


        List<FieldInfo> fields = new ArrayList<>();
        fields.add(new FieldInfo.Builder("invoice_group").type(FieldType.GROUP)
                    .child(new FieldInfo.Builder("invoice_number")
                        .type(FieldType.INVOICE_TYPE)
                        .build())
                    .child(new FieldInfo.Builder("invoice_date")
                        .type(FieldType.INVOICE_DATE)
                        .build())
                .build());

        Map<String, Object> parameters = new HashMap<>();

        LocalTrainingConfiguration configuration = LocalTrainingConfiguration.builder()
                .inputDir(inputDirPath)
                .outputDir(outputDirPath)
                .fields(fields)
                .parameters(parameters)
                .addResource("dictionary", Paths.get("home/ie-example/dictionary/InvoiceNumbers.csv"))
                .build();

        ModelRunner.run(ExampleIeModel.class, configuration);
    }
}

View model training results

After successful model training, the output folder contains a training sub-folder with the following folders:

• output is the folder with the model output, for example, statistics or trained model artifacts.
• process is the folder with the model training details, for example, status files or training logs.
• work is the working folder for the model training, for example, feature or temporary files.

The trained model artifact is added to the <%output_folder%>/training/output/model folder.

Cluster model training

A model generated from an Archetype contains a special class for model training—ModelTrainingRunner.

Run ModelTrainingRunner

To make the runner work, follow the steps below:

1. Generate public and private SSH keys. You can use PuTTYgen as a key generation tool for creating SSH keys for PuTTY. It is similar to the ssh-keygen tool used in other SSH implementations.
2. Send your public key to your system administrator and request access to the cluster.
3. Provide the following information to MarathonTrainingConfiguration.builder():
   • Path to a folder with training set files
   • For Information Extraction use case, a list of the FieldInfo fields representing your IE configuration.
   • Provide your custom model Class to method run

Configure model training on Marathon cluster

The configuration class for model training on the Marathon cluster has the following parameters:

• Required:

  • inputDir: path to the folder with the training set files.
  • endpoint: URL to the cluster.
  • sshHost: name of the Apache Mesos master node.
  • sshKey: SSH key path.
  • clusterSharedWorkerDir: path to workers handled by the Worker Management Service. Defaults to opt/workfusion/vds-data/workers/app/.
  • clusterWorkingDir: path to Marathon and Mesos on the cluster where your training starts. Defaults to opt/workfusion/vds-data/eval/.
  • id: training ID. Set a unique and verbose name to avoid collisions.
  • fields: a list of the FieldInfo fields representing your IE configuration.

• Optional:

  • sshPort gets the SSH port. Default: 22.
  • sshKeyPassphrase is the SSH key passphrase, if your SSH key is password protected.
  • testSetDir is the path to the folder with the test set files.
  • sshUserName is the SSH username for the connection. Default: wfuser.
  • parameters: additional parameters (for example, Search Engine limits, additional training configuration, and so on).

Here is an example of a cluster model runner configuration:

import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.workfusion.vds.sdk.api.nlp.configuration.FieldInfo;
import com.workfusion.vds.sdk.api.nlp.configuration.FieldType;
import com.workfusion.vds.sdk.run.ModelRunner;
import com.workfusion.vds.sdk.run.config.MarathonTrainingConfiguration;

public class ModelTrainingRunner {

    public static void main(String[] args) throws Exception {
        System.setProperty("WORKFLOW_LOG_FOLDER", "./logs/");

        List<FieldInfo> fields = new ArrayList<>();
        fields.add(new FieldInfo.Builder("transaction_data").type(FieldType.DATE).build());

        Map<String, Object> parameters = new HashMap<>();

        ClusterTrainingConfiguration configuration = ClusterTrainingConfiguration.builder()
            .inputDir(Paths.get("training_set"))
            .testSetDir(Paths.get("test_set"))
            .endpoint("url_to_cluster")
            .sshHost("host")
            .sshUsername("user")
            .sshPort(22)
            .sshKey("ssh_private_key_path")
            .sshKeyPassphrase("ssh_password")
            .clusterSharedWorkerDir("wms_shared_folder")
            .clusterWorkingDir("cluster_working_dir")
            .id("trainingId")
            .parameters(parameters)
            .fields(fields)
            .build();
        ModelRunner.run(YourIeModel.class, configuration);
    }
}

View model training results

After a successful model training, the Marathon folder contains a training sub-folder with the following folders:

• output: folder with the model output (statistics, trained model artifact, and so on)
• process: folder with the model training details (status files, training logs, and so on)
• work: working folder for the model training (feature files, temporary files, and so on)

The trained model artifact is added to the following folder: training MM.dd-hh.mm/training/output/model.