Edit

Local model training

A model generated from an archetype contains a local runner to allow you to train the configured model. The runner is located at .../run/ModelTrainingRunner.java.

The ModelTrainingRunner class contains a single method. You need to provide a path to the dataset and a working directory, and then specify a list of FieldInfo for the Information Extraction use case.

When training is finished, the resulting model will be available at workingDir/training/output/model. This path should be passed to the execution runner to check model quality.

Expand to see the example

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.workfusion.vds.sdk.api.nlp.configuration.FieldInfo;
import com.workfusion.vds.sdk.api.nlp.configuration.FieldType;
import com.workfusion.vds.sdk.run.ModelRunner;
import com.workfusion.vds.sdk.run.config.LocalTrainingConfiguration;

public class ModelTrainingRunner {

    public static void main(String[] args) throws Exception {
        System.setProperty("WORKFLOW_LOG_FOLDER", "./logs/");

        //Configure input and output
        Path inputDirPath = Paths.get("TODO add path to folder with training set");
        Path outputDirPath = Paths.get("TODO add path to output folder");

        //Configure the fields according to your use case
        List<FieldInfo> fields = new ArrayList<>();
        fields.add(new FieldInfo.Builder("invoice_number")
            .type(FieldType.INVOICE_TYPE)
            .build());
        fields.add(new FieldInfo.Builder("total_amount")
            .type(FieldType.PRICE)
            .build());
        //Add parameters, if needed
        Map<String, Object> parameters = new HashMap<>();

        LocalTrainingConfiguration configuration = LocalTrainingConfiguration.builder()
                .inputDir(inputDirPath)
                .outputDir(outputDirPath)
                .fields(fields)
                .parameters(parameters)
                .build();
        //Run your model specified by the Hypermodel class and configuration 
        ModelRunner.run(TrainingIeModel.class, configuration);
    }
}

Run ModelTrainingRunner

To make the runner work, provide the following parameters:

1. Path to the folder with training set files: inputDirPath.

2. Path to the output folder where trained model artifacts will be stored: outputDirPath.

3. In case of Information Extraction, provide a list of FieldInfo fields which represent your IE configuration.

4. Provide your custom model class to ModelRunner.run().

5. (Optional) Add additional parameters (for example, HPO limits, additional training configuration, and so on).

Here is an example of a ModelTrainingRunner configuration for one field invoice_number and a corresponding type:

Expand to see the example

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.workfusion.vds.sdk.api.nlp.configuration.FieldInfo;
import com.workfusion.vds.sdk.api.nlp.configuration.FieldType;
import com.workfusion.vds.sdk.run.ModelRunner;
import com.workfusion.vds.sdk.run.config.LocalTrainingConfiguration;

public class ModelTrainingRunner {

    public static void main(String[] args) throws Exception {
        System.setProperty("WORKFLOW_LOG_FOLDER", "./logs/");

        Path inputDirPath = Paths.get("/home/ie-example/input");
        Path outputDirPath = Paths.get("/home/ie-example/output");

        List<FieldInfo> fields = new ArrayList<>();
        fields.add(new FieldInfo.Builder("invoice_number")
                .type(FieldType.INVOICE_NUMBER)
                .required(true)
                .multiValue(false)
                .build());
        
        Map<String, Object> parameters = new HashMap<>();

        LocalTrainingConfiguration configuration = LocalTrainingConfiguration.builder()
                .inputDir(inputDirPath)
                .outputDir(outputDirPath)
                .fields(fields)
                .parameters(parameters)
                .addResource("dictionary", Paths.get("home/ie-example/dictionary/InvoiceNumbers.csv"))
                .build();

        ModelRunner.run(ExampleIeModel.class, configuration);
    }
}

Here is an example of a ModelTrainingRunner configuration for a group field invoice_group and a corresponding type:

Expand to see the example

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.workfusion.vds.sdk.api.nlp.configuration.FieldInfo;
import com.workfusion.vds.sdk.api.nlp.configuration.FieldType;
import com.workfusion.vds.sdk.run.ModelRunner;
import com.workfusion.vds.sdk.run.config.LocalTrainingConfiguration;

public class ModelTrainingRunner {

    public static void main(String[] args) throws Exception {
        System.setProperty("WORKFLOW_LOG_FOLDER", "./logs/");

        Path inputDirPath = Paths.get("/home/ie-example/input");
        Path outputDirPath = Paths.get("/home/ie-example/output");

        
        List<FieldInfo> fields = new ArrayList<>();
        fields.add(new FieldInfo.Builder("invoice_group").type(FieldType.GROUP)
                    .child(new FieldInfo.Builder("invoice_number")
                        .type(FieldType.INVOICE_TYPE)
                        .build())
                    .child(new FieldInfo.Builder("invoice_date")
                        .type(FieldType.INVOICE_DATE)
                        .build())
                .build());
        
        Map<String, Object> parameters = new HashMap<>();

        LocalTrainingConfiguration configuration = LocalTrainingConfiguration.builder()
                .inputDir(inputDirPath)
                .outputDir(outputDirPath)
                .fields(fields)
                .parameters(parameters)
                .addResource("dictionary", Paths.get("home/ie-example/dictionary/InvoiceNumbers.csv"))
                .build();

        ModelRunner.run(ExampleIeModel.class, configuration);
    }
}

Model training results

After a successful model training your output folder should contain a training sub-folder with following folders:

• output: A folder with model output (statistics, trained model artifact, and so on).
• process: A folder with model training details (status files, training logs, and so on).
• work: A working folder for model training (feature files, temporary files, and so on).

The trained model artifact will be placed in the following folder: <%output_folder%>/training/output/model.