Call VDS services from Bot Tasks
Call training for ML from machine config
Problem: Need to launch a training manually from a machine config.
Solution: A machine config with parameters:
Config example
<config>
<log message="attribute message, level=warn" level="WARN"/>
<script>
<![CDATA[
import javax.naming.Context;
import javax.naming.InitialContext;
import com.freedomoss.crowdcontrol.webharvest.RunDto;
import com.freedomoss.crowdcontrol.webharvest.WebHarvestTaskItem;
import com.freedomoss.crowdcontrol.webharvest.nlp.AnswerCompareUtils;
import com.freedomoss.crowdcontrol.webharvest.plugin.datastore.dto.AnswerInfoDTO;
import com.freedomoss.crowdcontrol.webharvest.plugin.datastore.dto.AutomationInfoDto;
import com.freedomoss.workfusion.utils.gson.GsonUtils;
import com.google.gson.GsonBuilder;
import com.google.gson.Gson;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.StringRequestEntity;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.validator.routines.RegexValidator;
import org.apache.commons.validator.routines.UrlValidator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Logger logger = LoggerFactory.getLogger("com.freedomoss.crowdcontrol.webharvest.AUTOMATION_LOG");
Gson GSON_WITHOUT_HTML_ESCAPING = new GsonBuilder().disableHtmlEscaping().create();
String extractUrl = "https://vds.workfusion.com/gateway-service/startEval";
Map buildRequestParamMap() {
Map requestParam = new HashMap();
requestParam.put("experimentGroupId", "document_xml_link_456a3-4838-10458696-hlushkou");
requestParam.put("modelName", "hk-dividends-information-extraction");
requestParam.put("modelVersion", "1.1");
requestParam.put("answers","[ {\"answerCode\" : \"non_latin\", \"answerType\" : \"CHECK_ONE\", \"subAnswers\" : [], \"required\" : false }, {\"answerCode\": \"document_xml_link\",\"answerType\": \"INFO_EXTRACTION\",\"subAnswers\": [{ \"answerCode\": \"ex_date\",\"answerType\": \"DATE\"}, {\"answerCode\": \"payment_date\",\"answerType\": \"DATE\"}, {\"answerCode\": \"record_date\",\"answerType\": \"DATE\"}, {\"answerCode\": \"gross_amount\",\"answerType\": \"FREE_TEXT\"}]}]");
return requestParam;
}
HttpClient buildHttpClient(String extractUrl) {
HttpClient extractClient = new HttpClient();
extractClient.setTimeout(0);
URL aURL = new URL(extractUrl);
extractClient.getState().setCredentials(
new AuthScope(aURL.getHost(), aURL.getPort()),
new UsernamePasswordCredentials("login name ", " password ")
);
return extractClient;
}
PostMethod buildHttpPostMethod(String extractUrl, Map requestParam) {
logger.info("Extract URL: " + extractUrl);
PostMethod httpPost = new PostMethod(extractUrl);
httpPost.setDoAuthentication(true);
StringRequestEntity stringRequestEntity = new StringRequestEntity(GsonUtils.GSON.toJson(requestParam), "application/json", "UTF-8");
httpPost.setRequestEntity(stringRequestEntity);
return httpPost;
}
]]>
</script>
<script></script>
</config>
Call ML extraction using hard-coded experiment group ID
Initial parameters are as follows:
document_xml_link: a link to HTML document to extract information.A map defines all parameters related to a ML model version, experiment group, and so on:
requestParam.put("experimentGroup", "document_xml_link_0278c183-1fad-4bc3-8f6b-396cb2d99469"); requestParam.put("experimentId", "41f65acbd25cd6237b53a04e22436124"); requestParam.put("modelId", "dividends-information-extraction"); requestParam.put("modelVersion", "1.0");
Expected result: The document_xml_link_tagged link should have a tagged HTML with attributes from ML.
Config example
<!-- sample one -->
<config>
<var-def name="get_content">
<http url="${document_xml_link}" charset="UTF-8"/>
</var-def>
<log message="attribute message, level=warn" level="WARN"/>
<script>
<![CDATA[
import javax.naming.Context;
import javax.naming.InitialContext;
import com.freedomoss.crowdcontrol.webharvest.RunDto;
import com.freedomoss.crowdcontrol.webharvest.WebHarvestTaskItem;
import com.freedomoss.crowdcontrol.webharvest.nlp.AnswerCompareUtils;
import com.freedomoss.crowdcontrol.webharvest.plugin.datastore.dto.AnswerInfoDTO;
import com.freedomoss.crowdcontrol.webharvest.plugin.datastore.dto.AutomationInfoDto;
import com.freedomoss.workfusion.utils.gson.GsonUtils;
import com.google.gson.GsonBuilder;
import com.google.gson.Gson;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.StringRequestEntity;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.validator.routines.RegexValidator;
import org.apache.commons.validator.routines.UrlValidator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Logger logger = LoggerFactory.getLogger("com.freedomoss.crowdcontrol.webharvest.AUTOMATION_LOG");
Gson GSON_WITHOUT_HTML_ESCAPING = new GsonBuilder().disableHtmlEscaping().create();
String extractUrl = "https://vds.workfusion.com/gateway-service/extract";
String createDocumentJson(String documentContent) {
Map map2Json = new HashMap();
List list2Json = new ArrayList();
list2Json.add(documentContent);
map2Json.put("textParts", list2Json);
String documentJson = GSON_WITHOUT_HTML_ESCAPING.toJson(map2Json);
return documentJson;
}
Map buildRequestParamMap( String documentJson) {
Map requestParam = new HashMap();
requestParam.put("experimentGroup", "document_xml_link_0278c183-1fad-4bc3-8f6b-396cb2d99469");
requestParam.put("experimentId", "41f65acbd25cd6237b53a04e22436124");
requestParam.put("modelId", "dividends-information-extraction");
requestParam.put("modelVersion", "1.0");
requestParam.put("document", documentJson);
return requestParam;
}
HttpClient buildHttpClient(String extractUrl) {
HttpClient extractClient = new HttpClient();
extractClient.setTimeout(0);
URL aURL = new URL(extractUrl);
extractClient.getState().setCredentials(
new AuthScope(aURL.getHost(), aURL.getPort()),
new UsernamePasswordCredentials("login-user", "password") // user and password are removed, please ask DEV OPS team to provide info
);
return extractClient;
}
PostMethod buildHttpPostMethod(String extractUrl, Map requestParam) {
logger.info("Extract URL: " + extractUrl);
PostMethod httpPost = new PostMethod(extractUrl);
httpPost.setDoAuthentication(true);
StringRequestEntity stringRequestEntity = new StringRequestEntity(GsonUtils.GSON.toJson(requestParam), "application/json", "UTF-8");
httpPost.setRequestEntity(stringRequestEntity);
return httpPost;
}
]]>
</script>
<script>
<![CDATA[
String documentJson = createDocumentJson(StringEscapeUtils.unescapeXml(get_content.toString()));
logger.warn("***** documentJson ******************************** \n " + documentJson );
Map requestParam = buildRequestParamMap( documentJson);
PostMethod httpPost = buildHttpPostMethod(extractUrl, requestParam);
HttpClient extractClient = buildHttpClient(extractUrl);
int httpState = extractClient.executeMethod(httpPost);
String result = httpPost.getResponseBodyAsString();
Map resultMap = new HashMap();
resultMap.putAll(GSON_WITHOUT_HTML_ESCAPING.fromJson(result, Map.class));
sys.defineVariable("externalValue", resultMap.get("tagged-text"),true);
]]>
</script>
<var-def name="currentDate">
<script return="result"></script>
</var-def>
<var-def name="last_checked">
<script return="result"></script>
</var-def>
<var-def name="got_content">
<template>${externalValue}</template>
</var-def>
<var-def name="s3Links">
<s3 bucket="customer-bucket">
<s3-put-public path="text_from_ocr/text_${currentDate}/extracted_${last_checked}.html" content="${got_content}" content-type="text/html" content-disposition="inline" />
</s3>
</var-def>
<var-def name="searchResultsLink">
<template>${s3Links.toList().get(0).toString()}</template>
</var-def>
<export include-original-data="true">
<single-column name="document_xml_link_tagged" value='${searchResultsLink}'/>
</export>
</config>
<!-- Sample TWO --->
<?xml version="1.0" encoding="UTF-8"?>
<config>
<var-def name="extractUrl"><template>${_sys_automation_url}/extract</template></var-def>
<script>
<![CDATA[
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.validator.UrlValidator;
//Pattern URL_REGEX = Pattern.compile("^(https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]");
UrlValidator urlValidator = new UrlValidator(new String[] { "http", "https" });
boolean isURL(String word) {
//Matcher matcher = URL_REGEX.matcher(word);
//return matcher.find();
return urlValidator.isValid(word);
}
]]>
</script>
<var-def name="urlOrContent">
<script return="document">
<![CDATA[
import com.freedomoss.crowdcontrol.webharvest.plugin.datastore.dto.AnswerInfoDTO;
import com.freedomoss.crowdcontrol.webharvest.plugin.datastore.dto.AutomationInfoDto;
import com.freedomoss.crowdcontrol.webharvest.RunDto;
import com.freedomoss.crowdcontrol.webharvest.WebHarvestTaskItem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.StringRequestEntity;
import org.apache.commons.validator.UrlValidator;
import com.freedomoss.crowdcontrol.webharvest.nlp.AnswerCompareUtils;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
Logger logger = LoggerFactory.getLogger("com.freedomoss.crowdcontrol.webharvest.AUTOMATION_LOG");
RunDto run = (RunDto) ((WebHarvestTaskItem) item.getWrappedObject()).getRun();
AutomationInfoDto automationInfo = run.automationInfo;
String originalCampaignUuid = automationInfo.originalCampaignUuid;
String textVariableName = _sys_automation_text_answer_code.toString();
logger.info("textVariableName = " + textVariableName + ". RunUUID: " + run.getUuid());
try {
Map mapping = run.getCampaignMapping(originalCampaignUuid).get(originalCampaignUuid);
if (mapping != null) {
for (Map.Entry entry : mapping.entrySet()) {
if (textVariableName.equals(entry.getValue())) {
textVariableName = entry.getKey();
break;
}
}
}
} catch (Exception e) {
}
logger.info("newTextVariableName = " + textVariableName + ". RunUUID: " + run.getUuid());
String document = sys.getVar(textVariableName).toString();
]]>
</script>
</var-def>
<case>
<if condition="${isURL(urlOrContent.toString())}">
<var-def name="documentToPost">
<to-text parser="org.apache.tika.parser.html.HtmlParser"
handler="org.apache.tika.parser.html.BoilerpipeContentHandler"
extractor="de.l3s.boilerpipe.extractors.ArticleExtractor">
<http url="${urlOrContent}"/>
</to-text>
</var-def>
</if>
<else>
<var-def name="documentToPost">
<template>${urlOrContent.toString()}</template>
</var-def>
</else>
</case>
<script>
<![CDATA[
import com.freedomoss.crowdcontrol.webharvest.plugin.datastore.dto.AnswerInfoDTO;
import com.freedomoss.crowdcontrol.webharvest.plugin.datastore.dto.AutomationInfoDto;
import com.freedomoss.crowdcontrol.webharvest.RunDto;
import com.freedomoss.crowdcontrol.webharvest.WebHarvestTaskItem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.StringRequestEntity;
import org.apache.commons.validator.UrlValidator;
import com.freedomoss.crowdcontrol.webharvest.nlp.AnswerCompareUtils;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import java.net.URL;
import com.freedomoss.workfusion.utils.gson.GsonUtils;
Logger logger = LoggerFactory.getLogger("com.freedomoss.crowdcontrol.webharvest.AUTOMATION_LOG");
RunDto run = (RunDto) ((WebHarvestTaskItem) item.getWrappedObject()).getRun();
AutomationInfoDto automationInfo = run.automationInfo;
Gson gson = new Gson();
HttpClient client = new HttpClient();
URL aURL = new URL(extractUrl.toString());
client.getState().setCredentials(
new AuthScope(aURL.getHost(), aURL.getPort()),
new UsernamePasswordCredentials("username", "userpassowoer")
);
Map documentJson = new HashMap();
List parts = new ArrayList();
parts.add(documentToPost.toString());
documentJson.put("textParts", parts);
String documentText = gson.toJson(documentJson);
Map requestParam = new HashMap();
requestParam.put("experimentGroup", automationInfo.experimentGroup);
requestParam.put("experimentId", automationInfo.experimentId);
requestParam.put("document", documentText);
requestParam.put("modelId", automationInfo.modelType);
requestParam.put("modelVersion", automationInfo.modelVersion);
StringRequestEntity stringRequestEntity = new StringRequestEntity(GsonUtils.GSON.toJson(requestParam), "application/json", "UTF-8");
PostMethod httpPost = new PostMethod(extractUrl.toString());
httpPost.setDoAuthentication(true);
logger.info("request = " + gson.toJson(requestParam));
StringRequestEntity stringRequestEntity = new StringRequestEntity(GsonUtils.GSON.toJson(requestParam), "application/json", "UTF-8");
httpPost.setRequestEntity(stringRequestEntity);
int httpState = client.executeMethod(httpPost);
String result = httpPost.getResponseBodyAsString();
logger.info("result = " + result);
if (httpState != 200 && httpState != 500 && httpState != 504) {
throw new RuntimeException("Error!!! UIMA service unavailable. Http code = " + result);
}
Map operationResult = gson.fromJson(result, Map.class);
boolean isExtractSuccessful = true;
String category = "";
Double score = 0.0;
Map resultMap = operationResult;
if (httpState == 500 || httpState == 504) {
errorMessage = result;
logger.error("Error was occurred during extract fields. " + errorMessage);
isExtractSuccessful = false;
} else {
category = resultMap.get("category");
score = resultMap.get("score");
isExtractSuccessful = !(category == null || category.isEmpty());
}
if (isExtractSuccessful) {
automationInfo.automatedRecords++;
}
logger.info("isExtractSuccessful = " + isExtractSuccessful);
logger.info("Results : " + category + " (" + score + ")");
]]>
</script>
<export include-original-data="true">
<single-column name="_sys_ie_extract_success" value="${isExtractSuccessful}"/>
<single-column name="_sys_extract_http_state" value="${httpState}"/>
<single-column name="_sys_extract_error_message" value="${errorMessage}"/>
<single-column name="_sys_group_probability" value='${score}'/>
<single-column name="_sys_group" value='S-${category}'/>
<single-column name="${_sys_automation_ie_answer_code}" value='${category}'/>
</export>
</config>