package parallelDots; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.json.JSONObject; import com.hof.mi.data.Wire; import com.hof.mi.etl.ETLException; import com.hof.mi.etl.ETLStepCategory; import com.hof.mi.etl.data.ETLStepMetadataFieldBean; import com.hof.mi.etl.step.AbstractETLRowStep; import com.hof.mi.etl.step.ETLStepAPIVersion; import com.hof.mi.etl.step.definition.ui.ETLStepConfigPanel; import com.hof.mi.etl.step.definition.ui.ETLStepConfigSection; import com.hof.mi.etl.step.definition.ui.ETLStepPanels; import com.hof.parameters.CssDeclarationImpl; import com.hof.parameters.CssRule; import com.hof.parameters.CssRuleImpl; import com.hof.parameters.GeneralPanelOptions; import com.hof.parameters.InputType; import com.hof.parameters.ParameterImpl; import com.hof.parameters.ParameterPanelCollection; import com.hof.parameters.ParameterValidation; import com.hof.util.Const; import com.hof.util.UtilString; import com.hof.util.YFLogger; import paralleldots.ParallelDots; public class ParallelDotsStep extends AbstractETLRowStep { public int counter = 0; private static final String TEXT_FIELD = "TEXT_FIELD"; private static String API_KEY = "API_KEY"; private boolean errorThrown=false; private boolean initialized = false; ParallelDots pd = null; private static final YFLogger log = YFLogger.getLogger(ParallelDotsStep.class.getName()); private static final String POSITIVE_FIELD = "POSITIVE_FIELD"; private static final String NEUTRAL_FIELD = "NEUTRAL_FIELD"; private static final String NEGATIVE_FIELD = "NEGATIVE_FIELD"; @Override public String getOffImage(){ return "PHN2ZyBpZD0iTGF5ZXJfMSIgZGF0YS1uYW1lPSJMYXllciAxIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI0MCIgaGVpZ2h0PSI0MCIgdmlld0JveD0iMCAwIDQwIDQwIj4KICA8ZGVmcz4KICAgIDxzdHlsZT4KICAgICAgLmNscy0xIHsKICAgICAgICBmaWxsOiAjYmJiOwogICAgICB9"+ "CiAgICA8L3N0eWxlPgogIDwvZGVmcz4KICA8dGl0bGU+QXJ0Ym9hcmQgMTwvdGl0bGU+CiAgPGc+CiAgICA8cGF0aCBjbGFzcz0iY2xzLTEiIGQ9Ik0yMC41MjkzNCwxMy4xNzM3NkE4LjIyMjkxLDguMjIyOTEsMCwwLDEsMjAuNjY5LDE0LjY4OTRhNi42NTg2Nyw2LjY1ODY3LDAsMCwxLTYuOTI4NjQsNy4wNTUzMyw3"+ "LjIxNzY2LDcuMjE3NjYsMCwwLDEsMC0xNC40MzUzMiw2LjkyODY2LDYuOTI4NjYsMCwwLDEsNC44MjY4OSwxLjg4MTA2TDE2LjUzMjQsMTEuMjI1MDZ2LS4wMDQ2NGEzLjk1MjU3LDMuOTUyNTcsMCwwLDAtMi43OTIwNi0xLjA5MTY1LDQuMzk2NzcsNC4zOTY3NywwLDAsMCwwLDguNzkyMSwzLjY5ODY5LDMuNjk4Njks"+ "MCwwLDAsMy45MzQtMi45MzIxNWgtMy45MzR2LTIuODE1WiIvPgogICAgPHBhdGggY2xhc3M9ImNscy0xIiBkPSJNMjYuMjUzNywxOC4yNTUyN0E3LjIxNzY2LDcuMjE3NjYsMCwxLDAsMzMuNDc3MzIsMjUuNDczYTcuMjE0NzcsNy4yMTQ3NywwLDAsMC03LjIyMzYyLTcuMjE3NzRabS4wMDYsMTIuOTkxODJBNS43NzQx"+ "NSw1Ljc3NDE1LDAsMSwxLDMyLjAzNSwyNS40NzNhNS43NzM1Myw1Ljc3MzUzLDAsMCwxLTUuNzc1MzEsNS43NzQwOFptMi41MjU1OS02LjQ5NTIyYTEuMDgzMjgsMS4wODMyOCwwLDEsMC0xLjA4MzI2LTEuMDgzMywxLjA4MTQ3LDEuMDgxNDcsMCwwLDAsMS4wODMyNiwxLjA4MzNabS01LjA1MTE0LDBhMS4wODMyOCwx"+ "LjA4MzI4LDAsMSwwLTEuMDgzMy0xLjA4MzMsMS4wODMsMS4wODMsMCwwLDAsMS4wODMzLDEuMDgzM1ptMi41MjU1NSw0LjY5MDc4YTMuOTY3MTMsMy45NjcxMywwLDAsMCwzLjY4ODc0LTIuNTI1NTlIMjIuNTcwODdhMy45NjcxNiwzLjk2NzE2LDAsMCwwLDMuNjg4ODQsMi41MjU1OVptMCwwIi8+CiAgPC9nPgo8L3N2"+ "Zz4K"; } @Override public String getOnImage(){ return "PHN2ZyBpZD0iTGF5ZXJfMSIgZGF0YS1uYW1lPSJMYXllciAxIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI0MCIgaGVpZ2h0PSI0MCIgdmlld0JveD0iMCAwIDQwIDQwIj4KICA8ZGVmcz4KICAgIDxzdHlsZT4KICAgICAgLmNscy0xIHsKICAgICAgICBmaWxsOiAjZmZmCiAgICAgIH0K"+ "ICAgIDwvc3R5bGU+CiAgPC9kZWZzPgogIDx0aXRsZT5BcnRib2FyZCAxPC90aXRsZT4KICA8Zz4KICAgIDxwYXRoIGNsYXNzPSJjbHMtMSIgZD0iTTIwLjUyOTM0LDEzLjE3Mzc2QTguMjIyOTEsOC4yMjI5MSwwLDAsMSwyMC42NjksMTQuNjg5NGE2LjY1ODY3LDYuNjU4NjcsMCwwLDEtNi45Mjg2NCw3LjA1NTMzLDcu"+ "MjE3NjYsNy4yMTc2NiwwLDAsMSwwLTE0LjQzNTMyLDYuOTI4NjYsNi45Mjg2NiwwLDAsMSw0LjgyNjg5LDEuODgxMDZMMTYuNTMyNCwxMS4yMjUwNnYtLjAwNDY0YTMuOTUyNTcsMy45NTI1NywwLDAsMC0yLjc5MjA2LTEuMDkxNjUsNC4zOTY3Nyw0LjM5Njc3LDAsMCwwLDAsOC43OTIxLDMuNjk4NjksMy42OTg2OSww"+ "LDAsMCwzLjkzNC0yLjkzMjE1aC0zLjkzNHYtMi44MTVaIi8+CiAgICA8cGF0aCBjbGFzcz0iY2xzLTEiIGQ9Ik0yNi4yNTM3LDE4LjI1NTI3QTcuMjE3NjYsNy4yMTc2NiwwLDEsMCwzMy40NzczMiwyNS40NzNhNy4yMTQ3Nyw3LjIxNDc3LDAsMCwwLTcuMjIzNjItNy4yMTc3NFptLjAwNiwxMi45OTE4MkE1Ljc3NDE1"+ "LDUuNzc0MTUsMCwxLDEsMzIuMDM1LDI1LjQ3M2E1Ljc3MzUzLDUuNzczNTMsMCwwLDEtNS43NzUzMSw1Ljc3NDA4Wm0yLjUyNTU5LTYuNDk1MjJhMS4wODMyOCwxLjA4MzI4LDAsMSwwLTEuMDgzMjYtMS4wODMzLDEuMDgxNDcsMS4wODE0NywwLDAsMCwxLjA4MzI2LDEuMDgzM1ptLTUuMDUxMTQsMGExLjA4MzI4LDEu"+ "MDgzMjgsMCwxLDAtMS4wODMzLTEuMDgzMywxLjA4MywxLjA4MywwLDAsMCwxLjA4MzMsMS4wODMzWm0yLjUyNTU1LDQuNjkwNzhhMy45NjcxMywzLjk2NzEzLDAsMCwwLDMuNjg4NzQtMi41MjU1OUgyMi41NzA4N2EzLjk2NzE2LDMuOTY3MTYsMCwwLDAsMy42ODg4NCwyLjUyNTU5Wm0wLDAiLz4KICA8L2c+Cjwvc3Zn"+ "Pgo="; } @Override public String getOnImageMIMEType() { return Const.MIME_TYPE_SVG; } @Override public String getOffImageMIMEType() { return Const.MIME_TYPE_SVG; } @Override public String getDefaultName() { return "ParallelDots"; } @Override public String getDefaultDescription() { return getText("Perform sentiment analysis on text documents using the Google Natural Language API", "mi.text.transformation.step.googlesentiment.description"); } @Override public ETLStepCategory getStepCategory() { return ETLStepCategory.TRANSFORM; } @Override public Collection validate() { return null; } @Override protected ParameterPanelCollection generatePanelCollection() { ETLStepConfigPanel sentimentPanel = new ETLStepConfigPanel("sentimentPanel", getETLStepBean().getStepName()); GeneralPanelOptions sectionOptions = new GeneralPanelOptions(); Set cssRules = new HashSet(); CssRule css =new CssRuleImpl(null, false); css.addDeclaration(new CssDeclarationImpl("margin-left", "10px")); css.addDeclaration(new CssDeclarationImpl("margin-right", "10px")); css.addDeclaration(new CssDeclarationImpl("margin-top", "10px")); cssRules.add(css); sectionOptions.setCssRules(cssRules); ETLStepConfigSection sentimentSection =new ETLStepConfigSection("SentimentSection", null); sentimentSection.setParameterSectionClassName("sentimentSection"); sentimentSection.setSectionOptions(sectionOptions); sentimentPanel.addSection(sentimentSection); ParameterImpl p = new ParameterImpl(); p = new ParameterImpl(); p.setInputType(InputType.TEXTBOX); p.setProperty(API_KEY); p.addViewOption("width", "365px"); p.setName(getText("API Key", "mi.text.transformation.step.googlesentiment.api.key")); sentimentSection.addParameter(p); p = new ParameterImpl(); p.setProperty(TEXT_FIELD); p.setInputType(InputType.SELECT); p.addViewOption("combineValues", true); p.setName(getText("Text field","mi.text.transformation.step.googlesentiment.text.field")); p.addViewOption("width", "200px"); ParameterValidation valid=new ParameterValidation(); valid.setNotEmpty(true); p.setValidationRules(valid); p.addPossibleValue("", " - - " + getText("Select", "mi.text.select") + " - - "); for(ETLStepMetadataFieldBean field: getDefaultMetadataFields()) { String fieldname=field.getFieldName(); if ("TEXT".equals(field.getFieldType()) && field.getEtlStepMetadataFieldUUID()!=getStepOption(POSITIVE_FIELD) && field.getEtlStepMetadataFieldUUID()!=getStepOption(NEUTRAL_FIELD) && field.getEtlStepMetadataFieldUUID()!=getStepOption(NEGATIVE_FIELD)){ p.addPossibleValue(field.getEtlStepMetadataFieldUUID(),fieldname); } } sentimentSection.addParameter(p); sentimentSection.setMappedParameterDataObject(TEXT_FIELD,getStepOption(TEXT_FIELD)); sentimentSection.setMappedParameterDataObject(API_KEY,getStepOption(API_KEY)); ETLStepPanels sentimentPanelCollection = new ETLStepPanels(); sentimentPanelCollection.addPanel(sentimentPanel); return sentimentPanelCollection; } @Override public void setupGeneratedFields() { List defaultFields = getDefaultMetadataFields(); if (defaultFields.size() > 0 && getStepOption(POSITIVE_FIELD)==null){ ETLStepMetadataFieldBean field = defaultFields.get(0); ETLStepMetadataFieldBean positive = field.duplicate(); positive.setFieldType("NUMERIC"); positive.setFieldName("Positive"); addNewGeneratedField(positive, POSITIVE_FIELD); ETLStepMetadataFieldBean neutral = field.duplicate(); neutral.setFieldType("NUMERIC"); neutral.setFieldName("Neutral"); addNewGeneratedField(neutral, NEUTRAL_FIELD); ETLStepMetadataFieldBean negative = field.duplicate(); negative.setFieldType("NUMERIC"); negative.setFieldName("Negative"); addNewGeneratedField(negative, NEGATIVE_FIELD); } } @Override public Map getDefaultInternalOptions() { return null; } @Override public Map getValidatedStepOptions() { return getStepOptions(); } @Override protected boolean processWireData(List fields) throws ETLException, InterruptedException { if (!initialized) { String APIKey= getStepOption(API_KEY); pd = new ParallelDots(APIKey); if (pd==null) errorThrown=true; initialized=true; } if (errorThrown) { return false; } Wire positive = getWireForField(getStepOption(POSITIVE_FIELD)); Wire neutral = getWireForField(getStepOption(NEUTRAL_FIELD)); Wire negative = getWireForField(getStepOption(NEGATIVE_FIELD)); Wire textInputWire = getWireForField(getStepOption(TEXT_FIELD)); if (textInputWire.getValue()==null){ positive.send(null); neutral.send(null); negative.send(null); return true; } String textDocument=textInputWire.getValue().toString(); textDocument=textDocument.replaceAll("\\\"", ""); try { String response = pd.sentiment(textDocument); JSONObject jsonObj = new JSONObject(response); JSONObject probabilities = (JSONObject) jsonObj.get("probabilities"); neutral.send(Double.valueOf(probabilities.get("neutral").toString())); negative.send(Double.valueOf(probabilities.get("negative").toString())); positive.send(Double.valueOf(probabilities.get("positive").toString())); } catch (Exception e) { positive.send(null); neutral.send(null); negative.send(null); log.error(e,e); return true; } return true; } public void parseData(Map data) { } public static String getText(String englishText, String translationKey) { String text = UtilString.getResourceString(translationKey); if (text==null) return englishText; return text; } @Override public ETLStepAPIVersion getAPIVersion() { return ETLStepAPIVersion.V1; } }