Edit

View code examples

Creating NER Annotators

import java.util.ArrayList;
import java.util.List;

import com.google.common.collect.Lists;
import com.workfusion.vds.sdk.api.hypermodel.annotation.ModelConfiguration;
import com.workfusion.vds.sdk.api.hypermodel.annotation.Named;
import com.workfusion.vds.sdk.api.nlp.annotator.Annotator;
import com.workfusion.vds.sdk.api.nlp.configuration.IeConfigurationContext;
import com.workfusion.vds.sdk.api.nlp.model.Document;
import com.workfusion.vds.sdk.nlp.component.annotator.EntityBoundaryAnnotator;
import com.workfusion.vds.sdk.nlp.component.annotator.ner.AhoCorasickDictionaryNerAnnotator;
import com.workfusion.vds.sdk.nlp.component.annotator.ner.BaseRegexNerAnnotator;
import com.workfusion.vds.sdk.nlp.component.dictionary.CsvDictionaryKeywordProvider;
import com.workfusion.vds.sdk.nlp.component.dictionary.MultiDictionaryDataProvider;
import com.workfusion.vds.sdk.nlp.component.dictionary.StringDictionaryProvider;

@ModelConfiguration
public class AnnotatorModelConfiguration {

    @Named("annotators")
    public List<Annotator<Document>> getAnnotators(IeConfigurationContext context) {
        final List<Annotator<Document>> annotators = new ArrayList<>();
        annotators.add(new EntityBoundaryAnnotator())
        annotators.add(BaseRegexNerAnnotator.getJavaPatternRegexNerAnnotator("email", "\\b[\\w._%+-]+@[\\w.-]+\\.[a-zA-Z]{2,}+\\b"));
        annotators.add(BaseRegexNerAnnotator.getRe2jPatternRegexNerAnnotator("email", "\\b[\\w._%+-]+@[\\w.-]+\\.[a-zA-Z]{2,}+\\b"));
        annotators.add(new AhoCorasickDictionaryNerAnnotator("company",
                new StringDictionaryProvider(Lists.newArrayList("Company A", "Company B", "Company C"))));

        annotators.add(new AhoCorasickDictionaryNerAnnotator("company",
                new CsvDictionaryKeywordProvider(context.getResource("classpath:dictionary/company.txt"))));

        annotators.add(new AhoCorasickDictionaryNerAnnotator("company",
                new MultiDictionaryDataProvider(new CsvDictionaryKeywordProvider(context.getResource("classpath:dictionary/company1.txt")),
                        new CsvDictionaryKeywordProvider(context.getResource("classpath:dictionary/company2.txt")))));
        return annotators;
    }

}

Overrideing Generic Tokenizer

import com.workfusion.vds.nlp.hypermodel.ie.generic.config.GenericIeAnnotatorConfiguration;
import com.workfusion.vds.nlp.hypermodel.ie.generic.config.GenericIeHypermodelConfiguration;
import com.workfusion.vds.sdk.api.hypermodel.annotation.Import;
import com.workfusion.vds.sdk.api.hypermodel.annotation.ModelConfiguration;
import com.workfusion.vds.sdk.api.hypermodel.annotation.Named;
import com.workfusion.vds.sdk.api.nlp.annotator.Annotator;
import com.workfusion.vds.sdk.api.nlp.model.Document;
import com.workfusion.vds.sdk.nlp.component.annotator.tokenizer.SplitterTokenAnnotator;

@ModelConfiguration
@Import(configurations = {
        @Import.Configuration(value = GenericIeHypermodelConfiguration.class)
})
public class CustomModelConfiguration {

    @Named(GenericIeAnnotatorConfiguration.TOKENIZER_ANNOTATOR)
    public Annotator<Document> getTokenizer() {
        return new SplitterTokenAnnotator("(\\s+)");
    }

}

Describing token

import com.workfusion.vds.nlp.hypermodel.ie.generic.config.GenericIeAnnotatorConfiguration;
import com.workfusion.vds.nlp.hypermodel.ie.generic.config.GenericIeHypermodelConfiguration;
import com.workfusion.vds.sdk.api.hypermodel.annotation.Import;
import com.workfusion.vds.sdk.api.hypermodel.annotation.ModelConfiguration;
import com.workfusion.vds.sdk.api.hypermodel.annotation.Named;
import com.workfusion.vds.sdk.api.nlp.annotator.Annotator;
import com.workfusion.vds.sdk.api.nlp.model.Document;
import com.workfusion.vds.sdk.nlp.component.annotator.tokenizer.MatcherTokenAnnotator;

@ModelConfiguration
@Import(configurations = {
        @Import.Configuration(value = GenericIeHypermodelConfiguration.class)
})
public class CustomModelConfiguration {

    @Named(GenericIeAnnotatorConfiguration.TOKENIZER_ANNOTATOR)
    public Annotator<Document> getTokenizer() {
        return new MatcherTokenAnnotator("(\\w+)");
    }

}

Simple Annotator

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.workfusion.vds.sdk.api.nlp.annotation.OnDestroy;
import com.workfusion.vds.sdk.api.nlp.annotation.OnInit;
import com.workfusion.vds.sdk.api.nlp.annotator.Annotator;
import com.workfusion.vds.sdk.api.nlp.model.Document;
import com.workfusion.vds.sdk.api.nlp.model.Token;

public class MyTokenAnnotator implements Annotator<Document> {

    private static final Pattern TOKEN_PATTERN = Pattern.compile("\\w+");

    @OnInit
    public void init() {
        //initialization method, executed once after constructor
    }

    @Override
    public void process(Document document) {
        final Matcher matcher = TOKEN_PATTERN.matcher(document.getText());
        while (matcher.find()) {
            int start = matcher.start();
            int end = matcher.end();
            document.add(Token.descriptor()
                    .setBegin(start)
                    .setEnd(end)
            );
        }

    }

    @OnDestroy
    public void destroy() {
        //destroy method, executed once after documents processing is finished
    }
}

Simple FE

import java.util.Collection;
import java.util.Collections;
import java.util.Map;

import com.workfusion.vds.sdk.api.nlp.annotation.OnDestroy;
import com.workfusion.vds.sdk.api.nlp.annotation.OnDocumentComplete;
import com.workfusion.vds.sdk.api.nlp.annotation.OnDocumentStart;
import com.workfusion.vds.sdk.api.nlp.annotation.OnInit;
import com.workfusion.vds.sdk.api.nlp.fe.Feature;
import com.workfusion.vds.sdk.api.nlp.fe.FeatureExtractor;
import com.workfusion.vds.sdk.api.nlp.model.Document;
import com.workfusion.vds.sdk.api.nlp.model.Element;

public class MyCoveredTextFE<T extends Element> implements FeatureExtractor<T> {

    @OnInit
    public void init(Map<String, String> parameters) {

    }

    @OnDocumentStart
    public void documentStart(Document document, Class<T> focusElementClass) {
    }

    @OnDocumentComplete
    public void documentComplete() {

    }

    @OnDestroy
    public void destroy() {

    }

    @Override
    public Collection<Feature> extract(Document document, T element) {
        return Collections.singleton(new Feature("CvrdTxt_" + element.getText(), 1.0));
    }

}

Post-Processor to change extracted value

import java.util.Collection;

import com.workfusion.vds.sdk.api.nlp.model.Field;
import com.workfusion.vds.sdk.api.nlp.model.IeDocument;
import com.workfusion.vds.sdk.api.nlp.normalization.NumberNormalizer;
import com.workfusion.vds.sdk.api.nlp.processing.Processor;
import com.workfusion.vds.sdk.nlp.component.processing.normalization.OcrAmountNormalizer;

public class ChangeValuePostProcessor implements Processor<IeDocument> {

    private final NumberNormalizer normalizer = new OcrAmountNormalizer();

    @Override
    public void process(IeDocument document) {
        Collection<Field> amounts = document.findFields("total_amount");
        amounts.forEach(a -> {
            String normalizedValue = normalizer.normalize(a.getValue(), "#.00");
            a.setValue(normalizedValue);
        });
    }
}

Post-Processor to change or remove extracted value

import java.math.BigDecimal;
import java.util.Collection;

import com.workfusion.vds.sdk.api.nlp.model.Field;
import com.workfusion.vds.sdk.api.nlp.model.IeDocument;
import com.workfusion.vds.sdk.api.nlp.processing.Processor;

public class RemoveFieldPostProcessor implements Processor<IeDocument> {

    @Override
    public void process(IeDocument document) {
        Collection<Field> amounts = document.findFields("total_amount");
        amounts.stream()
                .filter(a -> a.getScore().compareTo(BigDecimal.valueOf(0.5)) < 0)
                .forEach(document::remove);
    }
}

Post-Processor to add new field value

import com.workfusion.vds.sdk.api.nlp.model.Field;
import com.workfusion.vds.sdk.api.nlp.model.IeDocument;
import com.workfusion.vds.sdk.api.nlp.processing.Processor;

/**
Can be useful when you have the reference data.
**/
public class AddFieldPostProcessor implements Processor<IeDocument> {

    @Override
    public void process(IeDocument document) {
        document.add(
                Field.descriptor()
                        .setName("person_id")
                        .setValue("XSF12312")
                        .setScore(1.0));
    }
}

Importing fixed configuration

import com.workfusion.vds.nlp.hypermodel.ie.generic.config.GenericIeHypermodelConfiguration;
import com.workfusion.vds.sdk.api.hypermodel.annotation.Filter;
import com.workfusion.vds.sdk.api.hypermodel.annotation.Import;
import com.workfusion.vds.sdk.api.hypermodel.annotation.ModelConfiguration;

@ModelConfiguration
@Import(configurations = {
        @Import.Configuration(value = GenericIeHypermodelConfiguration.class),
},
        resources = {
                @Import.Resource(value = "/configuration/total_amount/config/parameters.json", condition = @Filter(expression = "field.code == 'total_amount'"))
        }
)
public class MyModelConfiguration {

}