valitydev/geck
14
0
Fork 0
mirror of https://github.com/valitydev/geck.git synced 2024-11-06 01:35:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Ft/bj 157/json processor (#47)

Browse Source 
* BJ-157: Added json processor; modified json-handler

* Added re-usable json-handler. Up pom version. Fixed readme.

* lol

* Fixed structure size, removed type identifiers for strings, added IllegalArgumentException catching

* Fixed bug with string escaping

* Fixed

* added test for checking work jsonprocessor with msgpackhandler
This commit is contained in:
Inal Arsanukaev 2017-05-19 15:31:39 +03:00 committed by GitHub
parent a29a79dde9
commit e65091b1f3
11 changed files with 226 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -3,7 +3,7 @@
![default](https://cloud.githubusercontent.com/assets/5084395/23034038/cf7e5eb0-f493-11e6-8698-66262306ca81.png)
### В комплекте ```нужное зачеркнуть```:
- serializer (msgpack, jolt, thrift, xml, ~~json~~)
- serializer (msgpack, jolt, thrift, xml, json)
- mock
- ~~migrator~~
- ~~filter~~

2
common/pom.xml
View File

@ -7,7 +7,7 @@
<parent>
<groupId>com.rbkmoney.geck</groupId>
<artifactId>parent</artifactId>
<version>0.2.0-SNAPSHOT</version>
<version>0.3.0-SNAPSHOT</version>
</parent>
<artifactId>common</artifactId>

2
pom.xml
View File

@ -12,7 +12,7 @@
<groupId>com.rbkmoney.geck</groupId>
<artifactId>parent</artifactId>
<version>0.2.0-SNAPSHOT</version>
<version>0.3.0-SNAPSHOT</version>
<packaging>pom</packaging>
<modules>

2
serializer/pom.xml
View File

@ -7,7 +7,7 @@
<parent>
<groupId>com.rbkmoney.geck</groupId>
<artifactId>parent</artifactId>
<version>0.2.0-SNAPSHOT</version>
<version>0.3.0-SNAPSHOT</version>
</parent>
<artifactId>serializer</artifactId>

4
serializer/src/main/java/com/rbkmoney/geck/serializer/Geck.java
View File

@ -6,7 +6,6 @@ import com.rbkmoney.geck.serializer.kit.tbase.TBaseProcessor;
import org.apache.thrift.TBase;
import java.io.IOException;
import java.io.StringWriter;
/**
* Created by tolkonepiu on 24/01/2017.
@ -15,9 +14,8 @@ public class Geck {
public String toJson(TBase src) {
try {
StringWriter writer = new StringWriter();
TBaseProcessor structProcessor = new TBaseProcessor();
JsonHandler jsonHandler = new JsonHandler(writer);
JsonHandler jsonHandler = new JsonHandler();
return structProcessor.process(src, jsonHandler).toString();
} catch (IOException ex) {
throw new RuntimeException(ex);

233
serializer/src/main/java/com/rbkmoney/geck/serializer/kit/json/JsonHandler.java
View File

@ -1,284 +1,153 @@
package com.rbkmoney.geck.serializer.kit.json;
import com.rbkmoney.geck.common.stack.ByteStack;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.rbkmoney.geck.serializer.StructHandler;
import com.rbkmoney.geck.serializer.exception.BadFormatException;
import gnu.trove.map.hash.THashMap;
import com.rbkmoney.geck.serializer.kit.StructType;
import java.io.IOException;
import java.io.StringWriter;
import java.io.Writer;
import java.util.Base64;
import java.util.Map;
import java.util.Objects;
/**
* Created by tolkonepiu on 27/01/2017.
*/
//TODO replace Writer to JsonNode or similar
//TODO pretty out (without type identifiers into staructure)
public class JsonHandler implements StructHandler<Writer> {
private final Map<Character, String> replaceCharsMap = new THashMap<>();
public static final String KEY = "key";
public static final String VALUE = "value";
public static final String ESC_SYMBOL = "@";
private Writer out;
private JsonGenerator jGenerator;
private JsonFactory jfactory = new JsonFactory();
{
for (int i = 0; i < '\u00A0'; i++) {
if (Character.getType((char) i) == Character.CONTROL) {
replaceCharsMap.put((char) i, String.format("\\u%04x", i));
}
}
replaceCharsMap.put('"', "\\\"");
replaceCharsMap.put('\n', "\\n");
replaceCharsMap.put('\r', "\\r");
replaceCharsMap.put('\b', "\\b");
replaceCharsMap.put('\f', "\\f");
replaceCharsMap.put('\t', "\\t");
replaceCharsMap.put('\\', "\\\\");
}
static final byte EMPTY_STRUCT = 1;
static final byte NONEMPTY_STRUCT = 2;
static final byte EMPTY_LIST = 3;
static final byte NONEMPTY_LIST = 4;
static final byte EMPTY_SET = 5;
static final byte NONEMPTY_SET = 6;
static final byte EMPTY_MAP = 7;
static final byte NONEMPTY_MAP = 8;
static final byte JSON_NAME = 9;
private final Writer out;
private final boolean pretty;
private ByteStack stack = new ByteStack();
public JsonHandler() {
this(new StringWriter());
}
public JsonHandler(Writer out) {
this(out, false);
}
public JsonHandler(Writer out, boolean pretty) {
Objects.requireNonNull(out, "Writer must not be null");
this.out = out;
this.pretty = pretty;
}
private void newline() throws IOException {
if (pretty) {
out.write("\n");
for (int i = 0; i < stack.size(); i++) {
out.write(' ');
}
try {
init();
} catch (BadFormatException e) {
throw new RuntimeException(e);//TODO
}
}
private void beforeValue() throws IOException {
if (stack.isEmpty()) {
return;
}
switch (stack.peek()) {
case EMPTY_LIST:
stack.pop();
stack.push(NONEMPTY_LIST);
newline();
break;
case EMPTY_SET:
stack.pop();
stack.push(NONEMPTY_SET);
newline();
break;
case EMPTY_MAP:
stack.pop();
stack.push(NONEMPTY_MAP);
newline();
break;
case NONEMPTY_LIST:
case NONEMPTY_SET:
case NONEMPTY_MAP:
out.append(',');
newline();
break;
case JSON_NAME:
out.append(':');
stack.pop();
stack.push(NONEMPTY_STRUCT);
break;
}
}
private void writeString(String value) throws IOException {
out.write('"');
int length = value.length();
for (int i = 0; i < length; i++) {
char charValue = value.charAt(i);
if (replaceCharsMap.containsKey(charValue)) {
out.write(replaceCharsMap.get(charValue));
} else {
out.write(charValue);
}
}
out.write('"');
}
private void writeBegin(byte empty, char symbol) throws IOException {
beforeValue();
stack.push(empty);
out.write(symbol);
}
private void writeEnd(byte empty, byte nonEmpty, char symbol) throws IOException {
byte element = stack.peek();
if (element != empty && element != nonEmpty) {
throw new BadFormatException();
}
if (element == nonEmpty) {
newline();
}
stack.pop();
out.write(symbol);
}
private void writeValue(String value) throws IOException {
writeValue(value, false);
}
private void writeValue(String value, boolean asString) throws IOException {
beforeValue();
if (asString) {
writeString(value);
} else {
out.write(value);
private void init() throws BadFormatException {
try {
out = new StringWriter();
jGenerator = jfactory.createGenerator(out);
} catch (IOException e) {
throw new BadFormatException("Unknown error when init", e);
}
}
@Override
public void beginStruct(int size) throws IOException {
writeBegin(EMPTY_STRUCT, '{');
jGenerator.writeStartObject();
}
@Override
public void endStruct() throws IOException {
writeEnd(EMPTY_STRUCT, NONEMPTY_STRUCT, '}');
jGenerator.writeEndObject();
}
@Override
public void beginList(int size) throws IOException {
writeBegin(EMPTY_LIST, '[');
jGenerator.writeStartArray();
jGenerator.writeString(StructType.LIST.getKey());
}
@Override
public void endList() throws IOException {
writeEnd(EMPTY_LIST, NONEMPTY_LIST, ']');
jGenerator.writeEndArray();
}
@Override
public void beginSet(int size) throws IOException {
writeBegin(EMPTY_SET, '[');
jGenerator.writeStartArray();
jGenerator.writeString(StructType.SET.getKey());
}
@Override
public void endSet() throws IOException {
writeEnd(EMPTY_SET, NONEMPTY_SET, ']');
jGenerator.writeEndArray();
}
@Override
public void beginMap(int size) throws IOException {
writeBegin(EMPTY_MAP, '[');
jGenerator.writeStartArray();
jGenerator.writeString(StructType.MAP.getKey());
}
@Override
public void endMap() throws IOException {
writeEnd(EMPTY_MAP, NONEMPTY_MAP, ']');
jGenerator.writeEndArray();
}
@Override
public void beginKey() throws IOException {
writeBegin(EMPTY_STRUCT, '{');
name("key");
jGenerator.writeStartObject();
name(KEY);
}
@Override
public void endKey() throws IOException {
if (stack.peek() == JSON_NAME) {
throw new BadFormatException();
}
}
@Override
public void beginValue() throws IOException {
name("value");
name(VALUE);
}
@Override
public void endValue() throws IOException {
writeEnd(EMPTY_STRUCT, NONEMPTY_STRUCT, '}');
jGenerator.writeEndObject();
}
@Override
public void name(String name) throws IOException {
Objects.requireNonNull(name, "JSON name must not be null");
if (stack.peek() == NONEMPTY_STRUCT) {
out.write(',');
}
newline();
stack.pop();
stack.push(JSON_NAME);
writeString(name);
jGenerator.writeFieldName(name);
}
@Override
public void value(boolean value) throws IOException {
writeValue(value ? "true" : "false");
jGenerator.writeBoolean(value);
}
@Override
public void value(String value) throws IOException {
writeValue(value, true);
if (value.startsWith(ESC_SYMBOL)) {
value = ESC_SYMBOL+value;
}
jGenerator.writeString(value);
}
@Override
public void value(double value) throws IOException {
writeValue(Double.toString(value));
jGenerator.writeNumber(value);
}
@Override
public void value(long value) throws IOException {
writeValue(Long.toString(value));
jGenerator.writeNumber(value);
}
@Override
public void value(byte[] value) throws IOException {
writeValue(Base64.getEncoder().encodeToString(value), true);
jGenerator.writeString(ESC_SYMBOL+Base64.getEncoder().encodeToString(value));
}
@Override
public void nullValue() throws IOException {
writeValue("null");
jGenerator.writeNull();
}
@Override
public Writer getResult() throws IOException {
out.close();
if (!stack.isEmpty()) {
throw new BadFormatException("stack is not empty");
}
return out;
jGenerator.close();
out.flush();
Writer resultOut = out;
init();
return resultOut;
}
}

112
serializer/src/main/java/com/rbkmoney/geck/serializer/kit/json/JsonProcessor.java Normal file
View File

@ -0,0 +1,112 @@
package com.rbkmoney.geck.serializer.kit.json;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.rbkmoney.geck.serializer.StructHandler;
import com.rbkmoney.geck.serializer.StructProcessor;
import com.rbkmoney.geck.serializer.exception.BadFormatException;
import com.rbkmoney.geck.serializer.kit.StructType;
import java.io.IOException;
import java.io.Writer;
import java.util.Base64;
import java.util.Iterator;
import java.util.Map;
/**
* Created by inalarsanukaev on 16.03.17.
*/
public class JsonProcessor implements StructProcessor<Writer> {
@Override
public <R> R process(Writer value, StructHandler<R> handler) throws IOException {
ObjectMapper mapper = new ObjectMapper();
JsonNode jsonNode = mapper.readTree(value.toString());
processNode(jsonNode, null, handler);
return handler.getResult();
}
private void processNode(JsonNode jsonNode, String name, StructHandler handler) throws IOException {
if (jsonNode != null) {
if (name != null) {
handler.name(name);
}
if (jsonNode.isNull()) {
handler.nullValue();
} else if (jsonNode.isBoolean()) {
handler.value(jsonNode.booleanValue());
} else if (jsonNode.isLong() || jsonNode.isInt()) {
handler.value(jsonNode.longValue());
} else if (jsonNode.isDouble()) {
handler.value(jsonNode.doubleValue());
} else if (jsonNode.isTextual()) {
String value = jsonNode.textValue();
if (value.startsWith(JsonHandler.ESC_SYMBOL)) {
String data = value.substring(1);
if (data.startsWith(JsonHandler.ESC_SYMBOL)) {
handler.value(data);
} else {
try {
handler.value(Base64.getDecoder().decode(data));
} catch (IllegalArgumentException e) {
throw new BadFormatException("Error when decode base64 field " + (name == null ? "" : name), e);
}
}
} else {
handler.value(value);
}
} else if (jsonNode.isObject()) {
handler.beginStruct(jsonNode.size());
processChildStructNodes(jsonNode.fields(), handler);
handler.endStruct();
} else if (jsonNode.isArray()) {
Iterator<JsonNode> elements = jsonNode.elements();
if (!elements.hasNext()) {
throw new BadFormatException("Incorrect structure of array. First element should exist!");
}
String arrCode = elements.next().textValue();
StructType arrType = StructType.valueOfKey(arrCode);
int size = jsonNode.size() - 1;
switch (arrType) {
case LIST:
handler.beginList(size);
processChildArrNodes(elements, handler);
handler.endList();
break;
case SET:
handler.beginSet(size);
processChildArrNodes(elements, handler);
handler.endSet();
break;
case MAP:
handler.beginMap(size);
while (elements.hasNext()) {
JsonNode mapEntry = elements.next();
handler.beginKey();
processNode(mapEntry.get(JsonHandler.KEY), null, handler);
handler.endKey();
handler.beginValue();
processNode(mapEntry.get(JsonHandler.VALUE), null, handler);
handler.endValue();
}
handler.endMap();
break;
default:
new BadFormatException("Unknown type of node: " + arrType + ". Must be on of them : " + StructType.LIST + ", " + StructType.SET + ", " + StructType.MAP);
}
}
}
}
private void processChildArrNodes(Iterator<JsonNode> elements, StructHandler handler) throws IOException {
while (elements.hasNext()) {
processNode(elements.next(), null, handler);
}
}
private void processChildStructNodes(Iterator<Map.Entry<String, JsonNode>> fields, StructHandler handler) throws IOException {
while (fields.hasNext()) {
Map.Entry<String, JsonNode> next = fields.next();
processNode(next.getValue(), next.getKey(), handler);
}
}
}

1
serializer/src/main/java/com/rbkmoney/geck/serializer/kit/xml/XMLProcessor.java
View File

@ -10,7 +10,6 @@ import org.w3c.dom.NodeList;
import javax.xml.transform.dom.DOMResult;
import java.io.IOException;
import java.sql.Struct;
import java.util.Arrays;
import java.util.Base64;

3
serializer/src/test/java/com/rbkmoney/geck/serializer/kit/damsel/DamselTest.java
View File

@ -26,11 +26,8 @@ import org.junit.Test;
import javax.xml.transform.*;
import javax.xml.transform.dom.DOMResult;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
import java.io.File;
import java.io.IOException;
import java.io.StringWriter;
import java.util.*;
/**

45
serializer/src/test/java/com/rbkmoney/geck/serializer/kit/json/JsonHandlerTest.java
View File

@ -1,45 +0,0 @@
package com.rbkmoney.geck.serializer.kit.json;
import com.rbkmoney.geck.serializer.kit.tbase.TBaseProcessor;
import com.rbkmoney.geck.serializer.test.TestObject;
import com.rbkmoney.geck.serializer.GeckTestUtil;
import org.json.JSONException;
import org.json.JSONObject;
import org.junit.Test;
import java.io.IOException;
/**
* Created by tolkonepiu on 15/02/2017.
*/
public class JsonHandlerTest {
@Test
public void jsonTest() throws JSONException, IOException {
TestObject testObject = GeckTestUtil.getTestObject();
String json = new TBaseProcessor().process(testObject, new JsonHandler()).toString();
System.out.println(json);
new JSONObject(json);
}
@Test
public void incorrectCharactersTest() throws IOException {
JsonHandler jsonHandler = new JsonHandler();
jsonHandler.beginStruct(2);
jsonHandler.name("ke\"k4 2\nqw\\eqw/eqw\nas\be");
char[] chars = new char['\u00A0'];
for (int i = 0; i < '\u00A0'; i++) {
chars[i] = (char) i;
}
jsonHandler.value(new String(chars));
jsonHandler.endStruct();
String json = jsonHandler.getResult().toString();
System.out.println(json);
new JSONObject(json);
}
}

58
serializer/src/test/java/com/rbkmoney/geck/serializer/kit/json/JsonTest.java Normal file
View File

@ -0,0 +1,58 @@
package com.rbkmoney.geck.serializer.kit.json;
import com.rbkmoney.damsel.v130.payment_processing.InvoicePaymentStarted;
import com.rbkmoney.geck.serializer.GeckTestUtil;
import com.rbkmoney.geck.serializer.kit.mock.MockTBaseProcessor;
import com.rbkmoney.geck.serializer.kit.msgpack.MsgPackHandler;
import com.rbkmoney.geck.serializer.kit.msgpack.MsgPackProcessor;
import com.rbkmoney.geck.serializer.kit.object.ObjectHandler;
import com.rbkmoney.geck.serializer.kit.object.ObjectProcessor;
import com.rbkmoney.geck.serializer.kit.tbase.TBaseHandler;
import com.rbkmoney.geck.serializer.kit.tbase.TBaseProcessor;
import com.rbkmoney.geck.serializer.test.TestObject;
import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
/**
* Created by iarsanukaev on 15/05/2017.
*/
public class JsonTest {
@Test
public void testInvoiceBackTransform() throws IOException {
InvoicePaymentStarted invoice1 = GeckTestUtil.getInvoicePaymentStarted();
InvoicePaymentStarted invoice2 =
new JsonProcessor().process(
new TBaseProcessor().process(invoice1, new JsonHandler()),
new TBaseHandler<>(InvoicePaymentStarted.class));
Assert.assertEquals(invoice1, invoice2);
}
@Test
public void testInvoiceBackTransform1() throws IOException {
InvoicePaymentStarted invoice1 = GeckTestUtil.getInvoicePaymentStarted();
InvoicePaymentStarted invoice2 =
MsgPackProcessor.newBinaryInstance().process(
new JsonProcessor().process(
new TBaseProcessor().process(
invoice1,
new JsonHandler()),
MsgPackHandler.newBufferedInstance(true)),
new TBaseHandler<>(InvoicePaymentStarted.class));
Assert.assertEquals(invoice1, invoice2);
}
@Test
public void jsonKebabTest() throws Exception {
TestObject testObject = new MockTBaseProcessor().process(new TestObject(), new TBaseHandler<>( TestObject.class));
JsonHandler handler = new JsonHandler();
String json1 = new TBaseProcessor().process(testObject, handler).toString();
System.out.println(json1);
//test re-use handler
String json2 = new TBaseProcessor().process(testObject, handler).toString();
System.out.println(json2);
Assert.assertEquals(json1, json2);
}
}