From e7bcf049e6804a12e617f1d66a5327d8d47166e8 Mon Sep 17 00:00:00 2001 From: Avishai Ish-Shalom Date: Mon, 3 Feb 2014 23:19:00 +0200 Subject: [PATCH 1/3] Add support for message objects, render object to json message --- .../net/logstash/log4j/JSONEventLayoutV0.java | 35 +++++++++++++++++++ .../net/logstash/log4j/JSONEventLayoutV1.java | 35 ++++++++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/logstash/log4j/JSONEventLayoutV0.java b/src/main/java/net/logstash/log4j/JSONEventLayoutV0.java index b29e3fe..50c3517 100644 --- a/src/main/java/net/logstash/log4j/JSONEventLayoutV0.java +++ b/src/main/java/net/logstash/log4j/JSONEventLayoutV0.java @@ -8,7 +8,12 @@ import org.apache.log4j.spi.LocationInfo; import org.apache.log4j.spi.LoggingEvent; import org.apache.log4j.spi.ThrowableInformation; +import sun.util.logging.resources.logging; +import java.io.Serializable; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.HashMap; import java.util.Map; import java.util.TimeZone; @@ -68,6 +73,10 @@ public String format(LoggingEvent loggingEvent) { logstashEvent.put("@source_host", hostname); logstashEvent.put("@message", loggingEvent.getRenderedMessage()); + Object messageObj = loggingEvent.getMessage(); + if (messageObj instanceof Serializable && !(messageObj instanceof String)) { + addObjectFieldData(messageObj); + } logstashEvent.put("@timestamp", dateFormat(timestamp)); if (loggingEvent.getThrowableInformation() != null) { @@ -103,6 +112,32 @@ public String format(LoggingEvent loggingEvent) { return logstashEvent.toString() + "\n"; } + private void addObjectFieldData(Object messageObj) { + Field[] fields = messageObj.getClass().getFields(); + Object value = null; + + for(Field f : fields) { + try { + value = f.get(messageObj); + if (value != null) fieldData.put(f.getName(), value); + } catch (IllegalAccessException e) { + } + } + Method[] methods = messageObj.getClass().getMethods(); + for(Method m : methods) + { + if(m.getName().startsWith("get")) + { + try { + value = m.invoke(messageObj); + } catch (IllegalAccessException e) { + } catch (InvocationTargetException e) { + } + if (value != null) fieldData.put(m.getName().substring(3), value); + } + } + } + public boolean ignoresThrowable() { return ignoreThrowable; } diff --git a/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java b/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java index aaf3228..9749615 100644 --- a/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java +++ b/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java @@ -10,6 +10,10 @@ import org.apache.log4j.spi.LoggingEvent; import org.apache.log4j.spi.ThrowableInformation; +import java.io.Serializable; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.HashMap; import java.util.Map; import java.util.TimeZone; @@ -103,8 +107,11 @@ public String format(LoggingEvent loggingEvent) { * Now we start injecting our own stuff. */ logstashEvent.put("source_host", hostname); + Object messageObject = loggingEvent.getMessage(); + if (messageObject instanceof Serializable && ! (messageObject instanceof String)) { + addObjectFieldData(messageObject); + } logstashEvent.put("message", loggingEvent.getRenderedMessage()); - if (loggingEvent.getThrowableInformation() != null) { final ThrowableInformation throwableInformation = loggingEvent.getThrowableInformation(); if (throwableInformation.getThrowable().getClass().getCanonicalName() != null) { @@ -184,4 +191,30 @@ private void addEventData(String keyname, Object keyval) { logstashEvent.put(keyname, keyval); } } + + private void addObjectFieldData(Object messageObj) { + Field[] fields = messageObj.getClass().getFields(); + Object value = null; + + for(Field f : fields) { + try { + value = f.get(messageObj); + if (value != null) addEventData(f.getName(), value); + } catch (IllegalAccessException e) { + } + } + Method[] methods = messageObj.getClass().getMethods(); + for(Method m : methods) + { + if(m.getName().startsWith("get")) + { + try { + value = m.invoke(messageObj); + } catch (IllegalAccessException e) { + } catch (InvocationTargetException e) { + } + if (value != null) addEventData(m.getName().substring(3), value); + } + } + } } From c976d67adbb2c3dae3e39bb39253d67e59cce6b2 Mon Sep 17 00:00:00 2001 From: Avishai Ish-Shalom Date: Tue, 4 Feb 2014 00:17:46 +0200 Subject: [PATCH 2/3] Add toggle flag for rendering message object using reflection --- .../net/logstash/log4j/JSONEventLayoutV0.java | 29 +++++++++++++----- .../net/logstash/log4j/JSONEventLayoutV1.java | 30 +++++++++++++++---- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/src/main/java/net/logstash/log4j/JSONEventLayoutV0.java b/src/main/java/net/logstash/log4j/JSONEventLayoutV0.java index 50c3517..321374c 100644 --- a/src/main/java/net/logstash/log4j/JSONEventLayoutV0.java +++ b/src/main/java/net/logstash/log4j/JSONEventLayoutV0.java @@ -8,7 +8,6 @@ import org.apache.log4j.spi.LocationInfo; import org.apache.log4j.spi.LoggingEvent; import org.apache.log4j.spi.ThrowableInformation; -import sun.util.logging.resources.logging; import java.io.Serializable; import java.lang.reflect.Field; @@ -20,6 +19,7 @@ public class JSONEventLayoutV0 extends Layout { + private boolean renderObjectFields = false; private boolean locationInfo = false; private String tags; @@ -49,16 +49,19 @@ public static String dateFormat(long timestamp) { * in the log messages. */ public JSONEventLayoutV0() { - this(true); + this(true, false); } /** * Creates a layout that optionally inserts location information into log messages. * * @param locationInfo whether or not to include location information in the log messages. + * @param renderObjectFields whether or not to render the fields of the message object into the json when an object is logged to log4j. + * Rendering the fields is done using reflection and incurs a performance cost */ - public JSONEventLayoutV0(boolean locationInfo) { + public JSONEventLayoutV0(boolean locationInfo, boolean renderObjectFields) { this.locationInfo = locationInfo; + this.renderObjectFields = renderObjectFields; } public String format(LoggingEvent loggingEvent) { @@ -73,12 +76,15 @@ public String format(LoggingEvent loggingEvent) { logstashEvent.put("@source_host", hostname); logstashEvent.put("@message", loggingEvent.getRenderedMessage()); - Object messageObj = loggingEvent.getMessage(); - if (messageObj instanceof Serializable && !(messageObj instanceof String)) { - addObjectFieldData(messageObj); - } logstashEvent.put("@timestamp", dateFormat(timestamp)); + if (renderObjectFields) { + Object messageObj = loggingEvent.getMessage(); + if (messageObj instanceof Serializable && !(messageObj instanceof String)) { + addObjectFieldData(messageObj); + } + } + if (loggingEvent.getThrowableInformation() != null) { final ThrowableInformation throwableInformation = loggingEvent.getThrowableInformation(); if (throwableInformation.getThrowable().getClass().getCanonicalName() != null) { @@ -160,6 +166,15 @@ public void setLocationInfo(boolean locationInfo) { this.locationInfo = locationInfo; } + /** + * Set whether or not to render the fields of the message object into the json when an object is logged to log4j. + * Rendering the fields is done using reflection and incurs a performance cost + * @param renderObjectFields + */ + public void setRenderObjectFields(boolean renderObjectFields) { + this.renderObjectFields = renderObjectFields; + } + public void activateOptions() { activeIgnoreThrowable = ignoreThrowable; } diff --git a/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java b/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java index 9749615..b9ec51d 100644 --- a/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java +++ b/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java @@ -20,6 +20,7 @@ public class JSONEventLayoutV1 extends Layout { + private boolean renderObjectFields = false; private boolean locationInfo = false; private String customUserFields; @@ -51,15 +52,19 @@ public static String dateFormat(long timestamp) { * in the log messages. */ public JSONEventLayoutV1() { - this(true); + this(true, false); } /** * Creates a layout that optionally inserts location information into log messages. * * @param locationInfo whether or not to include location information in the log messages. + * @param renderObjectFields whether or not to render the fields of the message object into the json when an object is logged to log4j. + * Rendering the fields is done using reflection and incurs a performance cost + */ - public JSONEventLayoutV1(boolean locationInfo) { + public JSONEventLayoutV1(boolean locationInfo, boolean renderObjectFields) { + this.renderObjectFields = renderObjectFields; this.locationInfo = locationInfo; } @@ -107,11 +112,15 @@ public String format(LoggingEvent loggingEvent) { * Now we start injecting our own stuff. */ logstashEvent.put("source_host", hostname); - Object messageObject = loggingEvent.getMessage(); - if (messageObject instanceof Serializable && ! (messageObject instanceof String)) { - addObjectFieldData(messageObject); - } logstashEvent.put("message", loggingEvent.getRenderedMessage()); + + if (renderObjectFields) { + Object messageObject = loggingEvent.getMessage(); + if (messageObject instanceof Serializable && ! (messageObject instanceof String)) { + addObjectFieldData(messageObject); + } + } + if (loggingEvent.getThrowableInformation() != null) { final ThrowableInformation throwableInformation = loggingEvent.getThrowableInformation(); if (throwableInformation.getThrowable().getClass().getCanonicalName() != null) { @@ -166,6 +175,15 @@ public void setLocationInfo(boolean locationInfo) { this.locationInfo = locationInfo; } + /** + * Set whether or not to render the fields of the message object into the json when an object is logged to log4j. + * Rendering the fields is done using reflection and incurs a performance cost + * @param renderObjectFields + */ + public void setRenderObjectFields(boolean renderObjectFields) { + this.renderObjectFields = renderObjectFields; + } + public String getUserFields() { return customUserFields; } public void setUserFields(String userFields) { this.customUserFields = userFields; } From 102e45a73313beaa4321b36cbaecad964b073329 Mon Sep 17 00:00:00 2001 From: Avishai Ish-Shalom Date: Wed, 5 Feb 2014 09:23:12 +0200 Subject: [PATCH 3/3] Fixed too eager field/method fetch, add tests --- .../net/logstash/log4j/JSONEventLayoutV0.java | 14 ++++--- .../net/logstash/log4j/JSONEventLayoutV1.java | 15 ++++---- .../logstash/log4j/JSONEventLayoutV0Test.java | 21 +++++++++++ .../logstash/log4j/JSONEventLayoutV1Test.java | 37 ++++++++++++++----- 4 files changed, 66 insertions(+), 21 deletions(-) diff --git a/src/main/java/net/logstash/log4j/JSONEventLayoutV0.java b/src/main/java/net/logstash/log4j/JSONEventLayoutV0.java index 321374c..48d24f4 100644 --- a/src/main/java/net/logstash/log4j/JSONEventLayoutV0.java +++ b/src/main/java/net/logstash/log4j/JSONEventLayoutV0.java @@ -119,8 +119,8 @@ public String format(LoggingEvent loggingEvent) { } private void addObjectFieldData(Object messageObj) { - Field[] fields = messageObj.getClass().getFields(); - Object value = null; + Field[] fields = messageObj.getClass().getDeclaredFields(); + Object value; for(Field f : fields) { try { @@ -129,17 +129,17 @@ private void addObjectFieldData(Object messageObj) { } catch (IllegalAccessException e) { } } - Method[] methods = messageObj.getClass().getMethods(); + Method[] methods = messageObj.getClass().getDeclaredMethods(); for(Method m : methods) { if(m.getName().startsWith("get")) { try { value = m.invoke(messageObj); + if (value != null) fieldData.put(m.getName().substring(3), value); } catch (IllegalAccessException e) { } catch (InvocationTargetException e) { } - if (value != null) fieldData.put(m.getName().substring(3), value); } } } @@ -174,7 +174,11 @@ public void setLocationInfo(boolean locationInfo) { public void setRenderObjectFields(boolean renderObjectFields) { this.renderObjectFields = renderObjectFields; } - + + public boolean getRenderObjectFields() { + return renderObjectFields; + } + public void activateOptions() { activeIgnoreThrowable = ignoreThrowable; } diff --git a/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java b/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java index b9ec51d..4f7e806 100644 --- a/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java +++ b/src/main/java/net/logstash/log4j/JSONEventLayoutV1.java @@ -184,6 +184,10 @@ public void setRenderObjectFields(boolean renderObjectFields) { this.renderObjectFields = renderObjectFields; } + public boolean getRenderObjectFields() { + return renderObjectFields; + } + public String getUserFields() { return customUserFields; } public void setUserFields(String userFields) { this.customUserFields = userFields; } @@ -211,27 +215,24 @@ private void addEventData(String keyname, Object keyval) { } private void addObjectFieldData(Object messageObj) { - Field[] fields = messageObj.getClass().getFields(); - Object value = null; + Field[] fields = messageObj.getClass().getDeclaredFields(); for(Field f : fields) { try { - value = f.get(messageObj); - if (value != null) addEventData(f.getName(), value); + addEventData(f.getName(), f.get(messageObj)); } catch (IllegalAccessException e) { } } - Method[] methods = messageObj.getClass().getMethods(); + Method[] methods = messageObj.getClass().getDeclaredMethods(); for(Method m : methods) { if(m.getName().startsWith("get")) { try { - value = m.invoke(messageObj); + addEventData(m.getName().substring(3), m.invoke(messageObj)); } catch (IllegalAccessException e) { } catch (InvocationTargetException e) { } - if (value != null) addEventData(m.getName().substring(3), value); } } } diff --git a/src/test/java/net/logstash/log4j/JSONEventLayoutV0Test.java b/src/test/java/net/logstash/log4j/JSONEventLayoutV0Test.java index 969ccc7..96a6d07 100644 --- a/src/test/java/net/logstash/log4j/JSONEventLayoutV0Test.java +++ b/src/test/java/net/logstash/log4j/JSONEventLayoutV0Test.java @@ -12,6 +12,8 @@ import org.junit.Ignore; import org.junit.Test; +import java.io.Serializable; + /** * Created with IntelliJ IDEA. * User: jvincent @@ -158,6 +160,25 @@ public void testJSONEventHasThreadName() { Assert.assertNotNull("ThreadName value is missing", atFields.get("threadName")); } + @Test + public void testMessageObjectRendering() { + JSONEventLayoutV0 layout = (JSONEventLayoutV0) appender.getLayout(); + boolean prevRenderObjectFields = layout.getRenderObjectFields(); + layout.setRenderObjectFields(true); + logger.info(new Serializable() { + String test = "TEST"; + int testNum = 1123; + }); + String message = appender.getMessages()[0]; + Object obj = JSONValue.parse(message); + JSONObject jsonObject = (JSONObject) obj; + JSONObject atFields = (JSONObject) jsonObject.get("@fields"); + Assert.assertTrue(atFields.containsKey("testNum")); + Assert.assertTrue(atFields.get("testNum") instanceof Integer); + Assert.assertEquals(atFields.get("testNum"), 1123); + layout.setRenderObjectFields(prevRenderObjectFields); + } + @Test public void testJSONEventLayoutNoLocationInfo() { JSONEventLayoutV0 layout = (JSONEventLayoutV0) appender.getLayout(); diff --git a/src/test/java/net/logstash/log4j/JSONEventLayoutV1Test.java b/src/test/java/net/logstash/log4j/JSONEventLayoutV1Test.java index 96ad821..be92e07 100644 --- a/src/test/java/net/logstash/log4j/JSONEventLayoutV1Test.java +++ b/src/test/java/net/logstash/log4j/JSONEventLayoutV1Test.java @@ -3,15 +3,16 @@ import junit.framework.Assert; import net.minidev.json.JSONObject; import net.minidev.json.JSONValue; -import org.apache.log4j.*; -import org.apache.log4j.or.ObjectRenderer; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.MDC; +import org.apache.log4j.NDC; import org.junit.After; -import org.junit.Before; -import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; +import java.io.Serializable; import java.util.HashMap; /** @@ -69,7 +70,7 @@ public void testJSONEventLayoutHasUserFieldsFromProps() { Assert.assertTrue("Event is not valid JSON", JSONValue.isValidJsonStrict(message)); Object obj = JSONValue.parse(message); JSONObject jsonObject = (JSONObject) obj; - Assert.assertTrue("Event does not contain field 'field1'" , jsonObject.containsKey("field1")); + Assert.assertTrue("Event does not contain field 'field1'", jsonObject.containsKey("field1")); Assert.assertEquals("Event does not contain value 'value1'", "propval1", jsonObject.get("field1")); System.clearProperty(JSONEventLayoutV1.ADDITIONAL_DATA_PROPERTY); } @@ -125,7 +126,7 @@ public void testJSONEventLayoutUserFieldsPropOverride() { Assert.assertTrue("Event is not valid JSON", JSONValue.isValidJsonStrict(message)); Object obj = JSONValue.parse(message); JSONObject jsonObject = (JSONObject) obj; - Assert.assertTrue("Event does not contain field 'field1'" , jsonObject.containsKey("field1")); + Assert.assertTrue("Event does not contain field 'field1'", jsonObject.containsKey("field1")); Assert.assertEquals("Event does not contain value 'propval1'", "propval1", jsonObject.get("field1")); layout.setUserFields(prevUserData); @@ -144,6 +145,24 @@ public void testJSONEventLayoutHasKeys() { } } + @Test + public void testMessageObjectRendering() { + JSONEventLayoutV1 layout = (JSONEventLayoutV1) appender.getLayout(); + boolean prevRenderObjectFields = layout.getRenderObjectFields(); + layout.setRenderObjectFields(true); + logger.info(new Serializable() { + String test = "TEST"; + int testNum = 1123; + }); + String message = appender.getMessages()[0]; + Object obj = JSONValue.parse(message); + JSONObject jsonObject = (JSONObject) obj; + Assert.assertTrue(jsonObject.containsKey("testNum")); + Assert.assertTrue(jsonObject.get("testNum") instanceof Integer); + Assert.assertEquals(jsonObject.get("testNum"), 1123); + layout.setRenderObjectFields(prevRenderObjectFields); + } + @Test public void testJSONEventLayoutHasNDC() { String ndcData = new String("json-layout-test"); @@ -165,14 +184,14 @@ public void testJSONEventLayoutHasMDC() { JSONObject jsonObject = (JSONObject) obj; JSONObject mdc = (JSONObject) jsonObject.get("mdc"); - Assert.assertEquals("MDC is wrong","bar", mdc.get("foo")); + Assert.assertEquals("MDC is wrong", "bar", mdc.get("foo")); } @Test public void testJSONEventLayoutHasNestedMDC() { HashMap nestedMdc = new HashMap(); - nestedMdc.put("bar","baz"); - MDC.put("foo",nestedMdc); + nestedMdc.put("bar", "baz"); + MDC.put("foo", nestedMdc); logger.warn("I should have nested MDC data in my log"); String message = appender.getMessages()[0]; Object obj = JSONValue.parse(message);