Logging to elk

py/strato/common/log/configurelogging.py

@@ -13,6 +13,7 @@
 _name = None
 _registered_file_handles = dict()
 
+
 def logFilename(name):
     return '%s/%s%s' % (config.LOGS_DIRECTORY, name, config.LOGS_SUFFIX)
 
@@ -40,6 +41,16 @@ def configureLogging(name, forceDirectory=None, registerConfigurationReloadSigna
         hostname = subprocess.check_output('/bin/hostname').strip()
     if registerConfigurationReloadSignal:
         _configureLoggingSignalHandlers()
+
+    from py.strato.tests.monitors import loggerconfig
+    log=loggerconfig.LogStashLogger(loggerName="_logs", messageType="_logs", verbose=False)
+
+    _log = log.getLogger()
+    _log.setLevel(logging.INFO)
+
+    logging.root.addHandler(_log)
+
+
     logging.info("Logging started for '%(name)s' on '%(hostname)s'", dict(
         name=name, hostname=hostname))
 
@@ -60,7 +71,6 @@ def configureLogger(loggerName):
     outputFilename = "%s__%s" % (_name, loggerName)
     _configureOutputToFile(logging.getLogger(loggerName), outputFilename)
 
-
 def addFileHandler(name, path):
     fileHandler = logging.FileHandler(filename=path)
     formatter = logging.Formatter(

py/strato/common/log/lineparse.py

@@ -21,8 +21,13 @@ def seperateTimestamp(message, timestampFormat=DEFAULT_TIMESTAMP_REGEX):
 
 
 def translateToEpoch(timeStamp, timestampFormat=DEFAULT_TIMESTAMP_FORMAT):
+    ms = 0
+    if timestampFormat.endswith('%f'):  # handle milliseconds
+       separator = timestampFormat[-3]
+       if separator in timeStamp:
+          ms = timeStamp.rsplit(separator, 1)[1]
     timeObject = datetime.datetime.strptime(timeStamp, timestampFormat)
-    return calendar.timegm(timeObject. timetuple())
+    return calendar.timegm(timeObject.timetuple()) + float(ms)/1000
 
 
 def getTimezoneOffset():

py/strato/common/log/log2text.py

@@ -57,7 +57,7 @@ class Formatter(object):
     def __init__(self, relativeTime, withThreads, showFullPaths, minimumLevel, microsecondPrecision, noColors, utc=False, sinceTime=None, untilTime="01/01/2025"):
         try:
             self.configFile = yaml.load(open(LOG_CONFIG_FILE_PATH, 'r').read())
-            if self.configFile['defaultTimezone'] != None:
+            if self.configFile['defaultTimezone'] is not None:
                 self._localTimezoneOffset = self.configFile['defaultTimezone'] * EPOCH_HOUR
             else:
                 self._localTimezoneOffset = lineparse.getTimezoneOffset()
@@ -534,9 +534,15 @@ def minimumLevel(minLevel, noDebug):
         sys.exit(result)
 
     formatter = Formatter(
-        minimumLevel=minimumLevel(args.min_level, args.noDebug), relativeTime=args.relativeTime, noColors=args.noColors,
-        microsecondPrecision=args.microsecondPrecision, showFullPaths=args.showFullPaths,
-        withThreads=args.withThreads, utc=args.utc, sinceTime=args.since, untilTime=args.until)
+        minimumLevel=minimumLevel(args.min_level, args.noDebug),
+        relativeTime=args.relativeTime,
+        noColors=args.noColors,
+        microsecondPrecision=args.microsecondPrecision,
+        showFullPaths=args.showFullPaths,
+        withThreads=args.withThreads,
+        utc=args.utc,
+        sinceTime=args.since,
+        untilTime=args.until)
 
     def _exitOrderlyOnCtrlC(signal, frame):
         sys.exit(0)

py/strato/common/log/machinereadableformatter.py

@@ -1,10 +1,13 @@
 import simplejson
+import uuid
 
 
 class MachineReadableFormatter:
     _IGNORED_ATTRIBUTES = set(['relativeCreated', 'msecs', 'message', 'exc_info', 'startColor', 'endColor'])
+    session_id = uuid.uuid4()
 
     def format(self, record):
+        record.session_id = str(self.session_id)
         data = dict(record.__dict__)
         for attribute in self._IGNORED_ATTRIBUTES:
             if attribute in data: