mirror of
https://github.com/valitydev/deanonimus.git
synced 2024-11-06 02:05:16 +00:00
OPS-407: refactor search query, use keywords (#49)
* OPS-407: refactor search query, use keywords * OPS-407: refactor search query, use keywords * OPS-407: refactor search query, use keywords
This commit is contained in:
Anatolii Karlov
GitHub
parent
14fd1ef327
commit
4fdca4bf68
32
README.md
32
README.md
@ -1,2 +1,34 @@
|
||||
# deanonimus
|
||||
Сервис для полнотекстового поиска party
|
||||
|
||||
актуальные примеры с spring data elastic + opensearch
|
||||
|
||||
https://github.com/opensearch-project/spring-data-opensearch/blob/main/spring-data-opensearch-examples/spring-boot-gradle/src/main/java/org/opensearch/data/example/service/MarketplaceInitializer.java
|
||||
https://github.com/M-Razavi/Spring-Data-OpenSearch-Example
|
||||
|
||||
полезные ссылки
|
||||
https://opensearch.org/docs/latest/query-dsl/full-text/index/
|
||||
https://opensearch.org/docs/latest/query-dsl/full-text/multi-match/
|
||||
|
||||
сниппеты
|
||||
```java
|
||||
getMappingResponse.get("party").mappings().properties().get("id").text().fields().get("keyword").keyword()
|
||||
|
||||
search.hits().hits().get(0).source()
|
||||
|
||||
var criteria = QueryBuilders.boolQuery()
|
||||
.must(QueryBuilders.matchQuery("authorName", author))
|
||||
.must(QueryBuilders.matchQuery("title", title));
|
||||
|
||||
SearchRequest searchRequest = new SearchRequest("books");
|
||||
searchRequest.source().query(criteria);
|
||||
|
||||
try {
|
||||
SearchResponse response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
|
||||
return Arrays.stream(response.getHits().getHits())
|
||||
.map(hit -> new ObjectMapper().convertValue(hit.getSourceAsMap(), Book.class))
|
||||
.collect(Collectors.toList());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Error executing search", e);
|
||||
}
|
||||
```
|
||||
@ -1,9 +1,7 @@
|
||||
package dev.vality.deanonimus.constant;
|
||||
|
||||
public class OpenSearchConstants {
|
||||
|
||||
public static final String PARTY_INDEX = "party";
|
||||
public static final String SHOP_INDEX = "shops";
|
||||
public static final String WALLET_INDEX = "wallets";
|
||||
public static final String CONTRACT_INDEX = "contracts";
|
||||
public static final String CONTRACTOR_INDEX = "contractors";
|
||||
|
||||
}
|
||||
|
||||
@ -10,7 +10,7 @@ import org.opensearch.client.opensearch.core.SearchResponse;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import static dev.vality.deanonimus.constant.OpenSearchConstants.*;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
@ -18,24 +18,19 @@ import static dev.vality.deanonimus.constant.OpenSearchConstants.*;
|
||||
public class SearchDaoImpl implements SearchDao {
|
||||
|
||||
@Value("${data.response.limit}")
|
||||
Integer responseLimit;
|
||||
private Integer responseLimit;
|
||||
|
||||
private final OpenSearchClient openSearchClient;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@Override
|
||||
public SearchResponse<Party> searchParty(String text) {
|
||||
|
||||
BoolQuery queryBuilder = new BoolQuery.Builder()
|
||||
.should(searchPartyFields(text),
|
||||
searchShopFields(text),
|
||||
searchContractFields(text),
|
||||
searchContractorFields(text),
|
||||
searchWalletFields(text))
|
||||
var queryBuilder = new BoolQuery.Builder()
|
||||
.should(searchBestFields(text, keywords()),
|
||||
searchPhrasePrefix(text, fields()))
|
||||
.build();
|
||||
|
||||
return openSearchClient.search(s -> s
|
||||
return openSearchClient.search(
|
||||
s -> s
|
||||
.size(responseLimit)
|
||||
.query(new Query.Builder()
|
||||
.bool(queryBuilder)
|
||||
@ -43,72 +38,46 @@ public class SearchDaoImpl implements SearchDao {
|
||||
Party.class);
|
||||
}
|
||||
|
||||
private Query searchContractorFields(String text) {
|
||||
return new NestedQuery.Builder()
|
||||
.path(CONTRACTOR_INDEX)
|
||||
.query(new Query(new MultiMatchQuery.Builder()
|
||||
.fields("contractors.id",
|
||||
private List<String> keywords() {
|
||||
return List.of(
|
||||
"id.keyword",
|
||||
"contractors.id.keyword",
|
||||
"contractors.russianLegalEntityInn.keyword",
|
||||
"contractors.russianLegalEntityRussianBankAccount.keyword",
|
||||
"contracts.id.keyword",
|
||||
"shops.id.keyword",
|
||||
"wallets.id.keyword");
|
||||
}
|
||||
|
||||
private List<String> fields() {
|
||||
return List.of(
|
||||
"email",
|
||||
"contractors.registeredUserEmail",
|
||||
"contractors.russianLegalEntityRegisteredName",
|
||||
"contractors.russianLegalEntityInn",
|
||||
"contractors.russianLegalEntityRussianBankAccount",
|
||||
"contractors.internationalLegalEntityLegalName",
|
||||
"contractors.internationalLegalEntityTradingName")
|
||||
.query(text)
|
||||
.type(TextQueryType.Phrase)
|
||||
.build()))
|
||||
.scoreMode(ChildScoreMode.Sum)
|
||||
.build().query();
|
||||
}
|
||||
|
||||
private Query searchContractFields(String text) {
|
||||
return new NestedQuery.Builder()
|
||||
.path(CONTRACT_INDEX)
|
||||
.query(new Query(new MultiMatchQuery.Builder()
|
||||
.fields("contracts.id",
|
||||
"contractors.internationalLegalEntityTradingName",
|
||||
"contracts.legalAgreementId",
|
||||
"contracts.reportActSignerFullName")
|
||||
.query(text)
|
||||
.type(TextQueryType.Phrase)
|
||||
.build()))
|
||||
.scoreMode(ChildScoreMode.Sum)
|
||||
.build().query();
|
||||
"contracts.reportActSignerFullName",
|
||||
"shops.locationUrl",
|
||||
"shops.detailsName",
|
||||
"wallets.name");
|
||||
}
|
||||
|
||||
private Query searchPartyFields(String text) {
|
||||
private Query searchBestFields(String text, List<String> fields) {
|
||||
return new Query(new MultiMatchQuery.Builder()
|
||||
.fields("id",
|
||||
"email")
|
||||
.fields(fields)
|
||||
.query(text)
|
||||
.type(TextQueryType.Phrase)
|
||||
.type(TextQueryType.BestFields)
|
||||
.operator(Operator.Or)
|
||||
.build());
|
||||
}
|
||||
|
||||
private Query searchShopFields(String text) {
|
||||
return new NestedQuery.Builder()
|
||||
.path(SHOP_INDEX)
|
||||
.query(new Query(new MultiMatchQuery.Builder()
|
||||
.fields("shops.id",
|
||||
"shops.locationUrl",
|
||||
"shops.detailsName")
|
||||
private Query searchPhrasePrefix(String text, List<String> fields) {
|
||||
return new Query(new MultiMatchQuery.Builder()
|
||||
.fields(fields)
|
||||
.query(text)
|
||||
.type(TextQueryType.Phrase)
|
||||
.build()))
|
||||
.scoreMode(ChildScoreMode.Sum)
|
||||
.build().query();
|
||||
.type(TextQueryType.PhrasePrefix)
|
||||
.operator(Operator.Or)
|
||||
.build());
|
||||
}
|
||||
|
||||
private Query searchWalletFields(String text) {
|
||||
return new NestedQuery.Builder()
|
||||
.path(WALLET_INDEX)
|
||||
.query(new Query(new MultiMatchQuery.Builder()
|
||||
.fields("wallets.id",
|
||||
"wallets.name")
|
||||
.query(text)
|
||||
.type(TextQueryType.Phrase)
|
||||
.build()))
|
||||
.scoreMode(ChildScoreMode.Sum)
|
||||
.build().query();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -8,13 +8,19 @@ import dev.vality.deanonimus.handler.DeanonimusServiceHandler;
|
||||
import dev.vality.deanonimus.service.OpenSearchService;
|
||||
import lombok.SneakyThrows;
|
||||
import org.apache.thrift.TException;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.opensearch.client.opensearch.OpenSearchClient;
|
||||
import org.opensearch.client.opensearch.indices.DeleteIndexRequest;
|
||||
import org.opensearch.client.opensearch.indices.ExistsRequest;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
import static dev.vality.deanonimus.constant.OpenSearchConstants.PARTY_INDEX;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
public class ReadTest extends AbstractIntegrationTest {
|
||||
@ -42,6 +48,13 @@ public class ReadTest extends AbstractIntegrationTest {
|
||||
private static final String INN = "1234234123";
|
||||
private static final String ACCOUNT = "9999999999";
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws IOException {
|
||||
var indices = client.indices();
|
||||
if (indices.exists(new ExistsRequest.Builder().index(PARTY_INDEX).build()).value()) {
|
||||
indices.delete(new DeleteIndexRequest.Builder().index(PARTY_INDEX).build());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchByPartyId() throws TException {
|
||||
@ -54,6 +67,30 @@ public class ReadTest extends AbstractIntegrationTest {
|
||||
.anyMatch(partySearchHit -> partySearchHit.getParty().getEmail().contains(EMAIL)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchByPartyIdAdds() throws TException {
|
||||
var id = UUID.randomUUID().toString();
|
||||
var mail = "asd zxc fgh";
|
||||
givenParty(id, mail);
|
||||
refreshIndices();
|
||||
var searchHits = deanonimusServiceHandler.searchParty(mail);
|
||||
assertFalse(searchHits.isEmpty());
|
||||
assertTrue(searchHits.stream()
|
||||
.anyMatch(partySearchHit -> partySearchHit.getParty().getEmail().contains(mail)));
|
||||
searchHits = deanonimusServiceHandler.searchParty(id);
|
||||
assertFalse(searchHits.isEmpty());
|
||||
assertTrue(searchHits.stream()
|
||||
.anyMatch(partySearchHit -> partySearchHit.getParty().getEmail().contains(mail)));
|
||||
// match partial field
|
||||
searchHits = deanonimusServiceHandler.searchParty("asd z");
|
||||
assertFalse(searchHits.isEmpty());
|
||||
assertTrue(searchHits.stream()
|
||||
.anyMatch(partySearchHit -> partySearchHit.getParty().getEmail().contains(mail)));
|
||||
// for keyword match only full field
|
||||
searchHits = deanonimusServiceHandler.searchParty(id.substring(0, 8));
|
||||
assertTrue(searchHits.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchByPartyIdWithoutTokens() throws TException {
|
||||
givenParty(PARTY + "-test-kek", EMAIL + "1");
|
||||
@ -169,6 +206,34 @@ public class ReadTest extends AbstractIntegrationTest {
|
||||
.getRegisteredUser().getEmail().contains(EMAIL)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchByContractorEmailAdds() throws TException {
|
||||
var id = UUID.randomUUID().toString();
|
||||
var mail = "asd zxc fgh";
|
||||
var party = givenParty(PARTY, null);
|
||||
givenRegisteredUserContractor(party, id, mail);
|
||||
refreshIndices();
|
||||
var searchHits = deanonimusServiceHandler.searchParty(mail);
|
||||
assertFalse(searchHits.isEmpty());
|
||||
assertTrue(searchHits.stream()
|
||||
.anyMatch(partySearchHit -> partySearchHit.getParty().getContractors().get(id).getContractor()
|
||||
.getRegisteredUser().getEmail().contains(mail)));
|
||||
searchHits = deanonimusServiceHandler.searchParty(id);
|
||||
assertFalse(searchHits.isEmpty());
|
||||
assertTrue(searchHits.stream()
|
||||
.anyMatch(partySearchHit -> partySearchHit.getParty().getContractors().get(id).getContractor()
|
||||
.getRegisteredUser().getEmail().contains(mail)));
|
||||
// match partial field
|
||||
searchHits = deanonimusServiceHandler.searchParty("asd z");
|
||||
assertFalse(searchHits.isEmpty());
|
||||
assertTrue(searchHits.stream()
|
||||
.anyMatch(partySearchHit -> partySearchHit.getParty().getContractors().get(id).getContractor()
|
||||
.getRegisteredUser().getEmail().contains(mail)));
|
||||
// for keyword match only full field
|
||||
searchHits = deanonimusServiceHandler.searchParty(id.substring(0, 8));
|
||||
assertTrue(searchHits.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchByContractorRussianLegalEntityRegisteredNameWithOneWord() throws TException {
|
||||
Party party = givenParty(PARTY, null);
|
||||
@ -357,6 +422,7 @@ public class ReadTest extends AbstractIntegrationTest {
|
||||
openSearchService.updateParty(party);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void refreshIndices() {
|
||||
client.indices().refresh();
|
||||
|
||||
371
src/test/resources/index_model.json
Normal file
371
src/test/resources/index_model.json
Normal file
@ -0,0 +1,371 @@
|
||||
{
|
||||
"party": {
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"blocking": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"contractors": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"internationalLegalEntityActualAddress": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"internationalLegalEntityLegalName": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"internationalLegalEntityRegisteredAddress": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"internationalLegalEntityRegisteredNumber": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"internationalLegalEntityTradingName": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"legalEntity": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"partyId": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"registeredUserEmail": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"russianLegalEntityActualAddress": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"russianLegalEntityInn": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"russianLegalEntityPostAddress": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"russianLegalEntityRegisteredName": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"russianLegalEntityRegisteredNumber": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"russianLegalEntityRussianBankAccount": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"russianLegalEntityRussianBankBik": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"russianLegalEntityRussianBankName": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"russianLegalEntityRussianBankPostAccount": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"type": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"contracts": {
|
||||
"properties": {
|
||||
"contractorId": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"id": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"legalAgreementId": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"partyId": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"paymentInstitutionId": {
|
||||
"type": "long"
|
||||
},
|
||||
"status": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"termsId": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"email": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"id": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"shops": {
|
||||
"properties": {
|
||||
"accountCurrencyCode": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"accountGuarantee": {
|
||||
"type": "long"
|
||||
},
|
||||
"accountPayout": {
|
||||
"type": "long"
|
||||
},
|
||||
"accountSettlement": {
|
||||
"type": "long"
|
||||
},
|
||||
"blocking": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"categoryId": {
|
||||
"type": "long"
|
||||
},
|
||||
"contractId": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"detailsDescription": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"detailsName": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"id": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"locationUrl": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"payoutToolId": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"suspension": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"suspension": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user