diff --git a/src-db/database/sourcedata/AD_REF_LIST.xml b/src-db/database/sourcedata/AD_REF_LIST.xml --- a/src-db/database/sourcedata/AD_REF_LIST.xml +++ b/src-db/database/sourcedata/AD_REF_LIST.xml @@ -85,6 +85,17 @@ + + + + + + + + + + + diff --git a/src/org/openbravo/externaldata/integration/process/AsynchronousProcessor.java b/src/org/openbravo/externaldata/integration/process/AsynchronousProcessor.java --- a/src/org/openbravo/externaldata/integration/process/AsynchronousProcessor.java +++ b/src/org/openbravo/externaldata/integration/process/AsynchronousProcessor.java @@ -219,8 +219,8 @@ log.error("Unexpected error processing item", e); allItemsProcessed = false; try { - OBDal.getInstance().getConnection(true).rollback(); - } catch (SQLException e1) { + OBDal.getInstance().rollbackAndClose(); + } catch (Exception e1) { throw new OBException(OBMessageUtils.messageBD(UNEXPECTED_ERROR_KEY), DbUtility.getUnderlyingSQLException(e1)); } diff --git a/src/org/openbravo/externaldata/integration/process/string/IdBatchIterator.java b/src/org/openbravo/externaldata/integration/process/string/IdBatchIterator.java new file mode 100644 --- /dev/null +++ b/src/org/openbravo/externaldata/integration/process/string/IdBatchIterator.java @@ -0,0 +1,57 @@ +/* + ************************************************************************************ + * Copyright (C) 2018 Openbravo S.L.U. + * Licensed under the Openbravo Commercial License version 1.0 + * You may obtain a copy of the License at http://www.openbravo.com/legal/obcl.html + * or in the legal folder of this module distribution. + ************************************************************************************ + */ + +package org.openbravo.externaldata.integration.process.string; + +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +/** + * The class used to separate a list of IDs (Strings) in batches of a particular size. + */ +public class IdBatchIterator implements Iterator { + static final String ID_SEPARATOR = "\n"; + private List idList; + private int batchSize; + private int position; + private int length; + + public IdBatchIterator(List idList, int batchSize) { + this.idList = idList; + this.batchSize = batchSize; + this.position = 0; + this.length = idList.size(); + } + + @Override + public boolean hasNext() { + return (position < length); + } + + @Override + public String next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + int partialLength = Math.min(position + batchSize, length); + StringBuilder currentBatch = new StringBuilder(); + while (position < partialLength) { + currentBatch.append(idList.get(position) + ID_SEPARATOR); + position++; + } + return currentBatch.toString(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } +} \ No newline at end of file diff --git a/src/org/openbravo/externaldata/integration/process/string/IdItemIterator.java b/src/org/openbravo/externaldata/integration/process/string/IdItemIterator.java new file mode 100644 --- /dev/null +++ b/src/org/openbravo/externaldata/integration/process/string/IdItemIterator.java @@ -0,0 +1,54 @@ +/* + ************************************************************************************ + * Copyright (C) 2018 Openbravo S.L.U. + * Licensed under the Openbravo Commercial License version 1.0 + * You may obtain a copy of the License at http://www.openbravo.com/legal/obcl.html + * or in the legal folder of this module distribution. + ************************************************************************************ + */ + +package org.openbravo.externaldata.integration.process.string; + +import static org.openbravo.externaldata.integration.process.string.IdBatchIterator.ID_SEPARATOR; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +import org.apache.commons.lang.StringUtils; + +/** + * The class used to iterate along a batch of IDs. + */ +public class IdItemIterator implements Iterator { + private String[] ids; + private int length; + private int position; + + public IdItemIterator(String batchOfIds) { + if (StringUtils.isEmpty(batchOfIds)) { + length = 0; + } else { + ids = batchOfIds.split(ID_SEPARATOR); + length = ids.length; + } + position = 0; + } + + @Override + public boolean hasNext() { + return (position < length); + } + + @Override + public String next() { + if (position == length) { + throw new NoSuchElementException(); + } + return ids[position++]; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } +} \ No newline at end of file diff --git a/src/org/openbravo/externaldata/integration/process/string/StringAsynchronousProcessor.java b/src/org/openbravo/externaldata/integration/process/string/StringAsynchronousProcessor.java new file mode 100644 --- /dev/null +++ b/src/org/openbravo/externaldata/integration/process/string/StringAsynchronousProcessor.java @@ -0,0 +1,43 @@ +/* + ************************************************************************************ + * Copyright (C) 2018 Openbravo S.L.U. + * Licensed under the Openbravo Commercial License version 1.0 + * You may obtain a copy of the License at http://www.openbravo.com/legal/obcl.html + * or in the legal folder of this module distribution. + ************************************************************************************ + */ + +package org.openbravo.externaldata.integration.process.string; + +import java.util.Iterator; +import java.util.List; + +import org.openbravo.client.kernel.ComponentProvider; +import org.openbravo.externaldata.integration.process.AsynchronousProcessor; + +@ComponentProvider.Qualifier("STRING") +public class StringAsynchronousProcessor extends AsynchronousProcessor> { + static final String ID_SEPARATOR = "\n"; + + @Override + protected Iterator getDataBatcher() { + List data = dataProcessor.getDataFromRaw(); + return new IdBatchIterator(data, recordSize); + } + + @Override + protected String getBatchFromList(List items) { + StringBuilder errors = new StringBuilder(); + for (String error : items) { + errors.append(error + ID_SEPARATOR); + } + return errors.toString(); + } + + @Override + protected Iterator getItemIterator() { + String batch = requestLine.getLinedata(); + return new IdItemIterator(batch); + } + +} diff --git a/src/org/openbravo/externaldata/integration/process/string/StringDataProcessor.java b/src/org/openbravo/externaldata/integration/process/string/StringDataProcessor.java new file mode 100644 --- /dev/null +++ b/src/org/openbravo/externaldata/integration/process/string/StringDataProcessor.java @@ -0,0 +1,111 @@ +/* + ************************************************************************************ + * Copyright (C) 2018 Openbravo S.L.U. + * Licensed under the Openbravo Commercial License version 1.0 + * You may obtain a copy of the License at http://www.openbravo.com/legal/obcl.html + * or in the legal folder of this module distribution. + ************************************************************************************ + */ + +package org.openbravo.externaldata.integration.process.string; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang.StringUtils; +import org.openbravo.base.exception.OBException; +import org.openbravo.client.kernel.ComponentProvider; +import org.openbravo.externaldata.integration.process.DataProcessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@ComponentProvider.Qualifier("STRING") +public class StringDataProcessor extends DataProcessor> { + private static final Logger log = LoggerFactory.getLogger(StringDataProcessor.class); + + @SuppressWarnings("unchecked") + @Override + public List getDataFromRaw() { + if (processDataProcessor != null) { + return processDataProcessor.getDataFromRaw(); + } else if (rawData != null) { + if (rawData instanceof List && !isRawDataEmpty() && isStringObject()) { + return (List) rawData; + } else if (rawData instanceof String) { + return toList((String) rawData); + } else if (rawData instanceof File) { + try { + return covertToString((File) rawData); + } catch (IOException e) { + throw new OBException("Error reading file", e); + } + } else if (rawData instanceof InputStream) { + try { + return covertToString((InputStream) rawData); + } catch (IOException e) { + throw new OBException("Error reading inputstream", e); + } + } + } + log.error("Unable to load data from raw"); + log.error("rawData instanceof {}", rawData.getClass()); + log.error("Content of rawData {}", rawData); + throw new NotImplementedException(); + } + + @Override + protected String getRawDataAsString() { + if (processDataProcessor != null && processDataProcessor.isGetRawDataAsStringImplemented()) { + return processDataProcessor.getRawDataAsString(); + } + if (rawData instanceof InputStream) { + return toStringList(getDataFromRaw()); + } + log.error("Unable to parse rawData to a String."); + log.error("rawData instanceof {}", rawData.getClass()); + log.error("Content of rawData {}", rawData); + throw new NotImplementedException(); + } + + public static String toStringList(List rawData) { + String result = ""; + for (String item : rawData) { + if (!StringUtils.isEmpty(result)) { + result.concat(", "); + } + result.concat("'").concat(item).concat("'"); + } + return result; + } + + private List covertToString(File file) throws IOException { + String data = FileUtils.readFileToString(file); + return toList(data); + } + + private List covertToString(InputStream inputStream) throws IOException { + String data = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + return toList(data); + } + + private List toList(String data) { + List list = new ArrayList(); + list.add(data); + return list; + } + + private boolean isRawDataEmpty() { + return ((List) rawData).isEmpty(); + } + + private boolean isStringObject() { + return ((List) rawData).get(0) instanceof String; + } +} diff --git a/src/org/openbravo/externaldata/integration/process/string/StringSynchProcessor.java b/src/org/openbravo/externaldata/integration/process/string/StringSynchProcessor.java new file mode 100644 --- /dev/null +++ b/src/org/openbravo/externaldata/integration/process/string/StringSynchProcessor.java @@ -0,0 +1,26 @@ +/* + ************************************************************************************ + * Copyright (C) 2018 Openbravo S.L.U. + * Licensed under the Openbravo Commercial License version 1.0 + * You may obtain a copy of the License at http://www.openbravo.com/legal/obcl.html + * or in the legal folder of this module distribution. + ************************************************************************************ + */ + +package org.openbravo.externaldata.integration.process.string; + +import java.util.Iterator; +import java.util.List; + +import org.openbravo.client.kernel.ComponentProvider; +import org.openbravo.externaldata.integration.process.SynchronousProcessor; + +@ComponentProvider.Qualifier("STRING") +public class StringSynchProcessor extends SynchronousProcessor> { + + @Override + protected Iterator getItemIterator() { + return dataProcessor.getDataFromRaw().iterator(); + } + +}