From 0174048cf4f7be5be47864495862f3c0e9390454 Mon Sep 17 00:00:00 2001 From: vitaxa Date: Wed, 26 Jun 2019 15:15:22 +0300 Subject: [PATCH] add batch insert (#4) * add batch insert --- pom.xml | 14 ++- .../java/com/rbkmoney/dao/GenericDao.java | 11 ++ .../rbkmoney/dao/impl/AbstractGenericDao.java | 81 ++++++++++++- .../dao/impl/AbstractGenericDaoTest.java | 112 ++++++++++++++++++ 4 files changed, 216 insertions(+), 2 deletions(-) create mode 100644 test/java/com/rbkmoney/dao/impl/AbstractGenericDaoTest.java diff --git a/pom.xml b/pom.xml index dabea04..e8be9cf 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ jar db-common-lib - 0.0.1-SNAPSHOT + 0.0.2-SNAPSHOT UTF-8 @@ -50,6 +50,18 @@ ${geck.common.version} provided + + junit + junit + 4.12 + test + + + org.mockito + mockito-core + 2.28.2 + test + diff --git a/src/main/java/com/rbkmoney/dao/GenericDao.java b/src/main/java/com/rbkmoney/dao/GenericDao.java index 3284e7e..50215b9 100644 --- a/src/main/java/com/rbkmoney/dao/GenericDao.java +++ b/src/main/java/com/rbkmoney/dao/GenericDao.java @@ -66,4 +66,15 @@ public interface GenericDao { int execute(String namedSql, SqlParameterSource parameterSource, int expectedRowsAffected, NamedParameterJdbcTemplate namedParameterJdbcTemplate, KeyHolder keyHolder) throws DaoException; + long batchExecute(List queries) throws DaoException; + + long batchExecute(List queries, int expectedRowsPerQueryAffected) throws DaoException; + + long batchExecute(List queries, int expectedRowsPerQueryAffected, NamedParameterJdbcTemplate namedParameterJdbcTemplate) throws DaoException; + + long batchExecute(String namedSql, List parameterSources) throws DaoException; + + long batchExecute(String namedSql, List parameterSources, int expectedRowsPerQueryAffected) throws DaoException; + + long batchExecute(String namedSql, List parameterSources, int expectedRowsPerQueryAffected, NamedParameterJdbcTemplate namedParameterJdbcTemplate) throws DaoException; } diff --git a/src/main/java/com/rbkmoney/dao/impl/AbstractGenericDao.java b/src/main/java/com/rbkmoney/dao/impl/AbstractGenericDao.java index 4ea99c3..80ae99e 100644 --- a/src/main/java/com/rbkmoney/dao/impl/AbstractGenericDao.java +++ b/src/main/java/com/rbkmoney/dao/impl/AbstractGenericDao.java @@ -17,9 +17,12 @@ import org.springframework.jdbc.support.KeyHolder; import javax.sql.DataSource; import java.sql.Types; import java.time.LocalDateTime; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; public abstract class AbstractGenericDao extends NamedParameterJdbcDaoSupport implements GenericDao { @@ -217,6 +220,76 @@ public abstract class AbstractGenericDao extends NamedParameterJdbcDaoSupport im } } + @Override + public long batchExecute(List queries) throws DaoException { + return batchExecute(queries, -1); + } + + @Override + public long batchExecute(List queries, int expectedRowsAffected) throws DaoException { + return batchExecute(queries, expectedRowsAffected, getNamedParameterJdbcTemplate()); + } + + @Override + public long batchExecute(List queries, int expectedRowsAffected, NamedParameterJdbcTemplate namedParameterJdbcTemplate) throws DaoException { + AtomicLong affectedRowCounter = new AtomicLong(); + queries.stream() + .collect( + Collectors.groupingBy( + query -> query.getSQL(ParamType.NAMED), + LinkedHashMap::new, + Collectors.mapping(query -> toSqlParameterSource(query.getParams()), Collectors.toList()) + ) + ) + .forEach( + (namedSql, parameterSources) -> { + long affectedRowCount = batchExecute( + namedSql, + parameterSources, + expectedRowsAffected, + namedParameterJdbcTemplate + ); + affectedRowCounter.getAndAccumulate(affectedRowCount, Long::sum); + } + ); + return affectedRowCounter.get(); + } + + @Override + public long batchExecute(String namedSql, List parameterSources) throws DaoException { + return batchExecute(namedSql, parameterSources, -1); + } + + @Override + public long batchExecute(String namedSql, List parameterSources, int expectedRowsAffected) throws DaoException { + return batchExecute(namedSql, parameterSources, expectedRowsAffected, getNamedParameterJdbcTemplate()); + } + + @Override + public long batchExecute(String namedSql, List parameterSources, int expectedRowsAffected, NamedParameterJdbcTemplate namedParameterJdbcTemplate) throws DaoException { + try { + int[] rowsPerBatchAffected = namedParameterJdbcTemplate.batchUpdate(namedSql, parameterSources.toArray(new SqlParameterSource[0])); + + if (rowsPerBatchAffected.length != parameterSources.size()) { + throw new JdbcUpdateAffectedIncorrectNumberOfRowsException(namedSql, parameterSources.size(), rowsPerBatchAffected.length); + } + + int count = 0; + for (int i : rowsPerBatchAffected) { + count += i; + } + if (expectedRowsAffected != -1) { + if (count != expectedRowsAffected) { + throw new JdbcUpdateAffectedIncorrectNumberOfRowsException(namedSql, expectedRowsAffected, count); + } + } + + return count; + } catch (NestedRuntimeException ex) { + throw new DaoException(ex); + } + } + protected Condition appendDateTimeRangeConditions(Condition condition, Field field, Optional fromTime, @@ -235,7 +308,13 @@ public abstract class AbstractGenericDao extends NamedParameterJdbcDaoSupport im MapSqlParameterSource sqlParameterSource = new MapSqlParameterSource(); for (Map.Entry> entry : params.entrySet()) { Param param = entry.getValue(); - if (param.getValue() instanceof LocalDateTime || param.getValue() instanceof EnumType) { + Class type = param.getDataType().getType(); + if (String.class.isAssignableFrom(type)) { + String value = Optional.ofNullable(param.getValue()) + .map(stringValue -> ((String) stringValue).replace("\u0000", "\\u0000")) + .orElse(null); + sqlParameterSource.addValue(entry.getKey(), value); + } else if (LocalDateTime.class.isAssignableFrom(type) || EnumType.class.isAssignableFrom(type)) { sqlParameterSource.addValue(entry.getKey(), param.getValue(), Types.OTHER); } else { sqlParameterSource.addValue(entry.getKey(), param.getValue()); diff --git a/test/java/com/rbkmoney/dao/impl/AbstractGenericDaoTest.java b/test/java/com/rbkmoney/dao/impl/AbstractGenericDaoTest.java new file mode 100644 index 0000000..2113393 --- /dev/null +++ b/test/java/com/rbkmoney/dao/impl/AbstractGenericDaoTest.java @@ -0,0 +1,112 @@ +package com.rbkmoney.dao.impl; + +import com.rbkmoney.dao.DaoException; +import org.jooq.DataType; +import org.jooq.Param; +import org.jooq.Query; +import org.jooq.conf.ParamType; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.jdbc.core.namedparam.SqlParameterSource; + +import javax.sql.DataSource; +import java.sql.Types; +import java.time.LocalDateTime; +import java.util.*; + +import static org.mockito.Mockito.*; + +@RunWith(MockitoJUnitRunner.class) +public class AbstractGenericDaoTest { + + private TestDao testDaoSpy; + + @Before + public void setUp() throws Exception { + testDaoSpy = spy(new TestDao(mock(DataSource.class))); + } + + @Test + public void batchExecuteTest() { + final NamedParameterJdbcTemplate namedParameterJdbcTemplateMock = mock(NamedParameterJdbcTemplate.class); + int[] rowPerBatchAffected = { 1, 1 }; + when(namedParameterJdbcTemplateMock.batchUpdate(anyString(), any(SqlParameterSource[].class))).thenReturn(rowPerBatchAffected); + when(testDaoSpy.getNamedParameterJdbcTemplate()).thenReturn(namedParameterJdbcTemplateMock); + + final Map> firstParamMap = paramMapMock("testString", "testValue", String.class); + firstParamMap.putAll(paramMapMock("testDate", LocalDateTime.now(), LocalDateTime.class)); + + final Query firstQueryMock = mock(Query.class); + when(firstQueryMock.getSQL(ParamType.NAMED)).thenReturn("test sql"); + when(firstQueryMock.getParams()).thenReturn(firstParamMap); + + final Map> secondParamMap = paramMapMock("testInteger", 12345, Integer.class); + secondParamMap.putAll(paramMapMock("testLong", 12345L, Long.class)); + + final Query secondQueryMock = mock(Query.class); + when(secondQueryMock.getSQL(ParamType.NAMED)).thenReturn("test sql"); + when(secondQueryMock.getParams()).thenReturn(secondParamMap); + + final List queryList = Arrays.asList(firstQueryMock, secondQueryMock); + final long rowsAffected = testDaoSpy.batchExecute(queryList, 2); + Assert.assertEquals(2, rowsAffected); + } + + @Test(expected = DaoException.class) + public void batchExceptionTest() { + final NamedParameterJdbcTemplate namedParameterJdbcTemplateMock = mock(NamedParameterJdbcTemplate.class); + int[] rowPerBatchAffected = { 1 }; + when(namedParameterJdbcTemplateMock.batchUpdate(anyString(), any(SqlParameterSource[].class))).thenReturn(rowPerBatchAffected); + when(testDaoSpy.getNamedParameterJdbcTemplate()).thenReturn(namedParameterJdbcTemplateMock); + + final Map> firstParamMap = paramMapMock("testString", "testValue", String.class); + firstParamMap.putAll(paramMapMock("testDate", LocalDateTime.now(), LocalDateTime.class)); + + final Query firstQueryMock = mock(Query.class); + when(firstQueryMock.getSQL(ParamType.NAMED)).thenReturn("test sql"); + when(firstQueryMock.getParams()).thenReturn(firstParamMap); + + final List queryList = Collections.singletonList(firstQueryMock); + final long rowsAffected = testDaoSpy.batchExecute(queryList, 2); + } + + @Test + public void toSqlParameterSourceNullByteTest() { + final Map> paramMap = paramMapMock("test", "\u0000", String.class); + final SqlParameterSource sqlParameterSource = testDaoSpy.toSqlParameterSource(paramMap); + Assert.assertEquals("\\u0000", sqlParameterSource.getValue("test")); + } + + @Test + public void toSqlParameterSourceDateTest() { + final LocalDateTime dateTime = LocalDateTime.now(); + final Map> paramMap = paramMapMock("testDate", dateTime, LocalDateTime.class); + final SqlParameterSource sqlParameterSource = testDaoSpy.toSqlParameterSource(paramMap); + Assert.assertEquals(dateTime, sqlParameterSource.getValue("testDate")); + Assert.assertEquals(Types.OTHER, sqlParameterSource.getSqlType("testDate")); + } + + private Map> paramMapMock(String key, Object value, Class dataType) { + final Param paramMock = mock(Param.class); + final DataType dataTypeMock = mock(DataType.class); + when(dataTypeMock.getType()).thenReturn(dataType); + when(paramMock.getDataType()).thenReturn(dataTypeMock); + when(paramMock.getValue()).thenReturn(value); + final Map> paramMap = new HashMap<>(); + paramMap.put(key, paramMock); + + return paramMap; + } + + private class TestDao extends AbstractGenericDao { + + TestDao(DataSource dataSource) { + super(dataSource); + } + } + +}