feat: Autodetect timestamp

Author: tonyo

go.mod

@@ -5,6 +5,7 @@ go 1.12
 require (
 	github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect
 	github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect
+	github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/getsentry/sentry-go v0.1.1-0.20190624124141-69c26e4dfca8
 	github.com/hpcloud/tail v1.0.1-0.20180514194441-a1dbeea552b7

go.sum

@@ -2,6 +2,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5Vpd
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 h1:c4mLfegoDw6OhSJXTd2jUEQgZUQuJWtocudb97Qn9EM=
+github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=

grok-patterns.go

@@ -3,5 +3,7 @@ package main
 import "github.com/vjeantet/grok"
 
 func AddDefaultPatterns(g *grok.Grok) {
-	g.AddPattern("NGINX_ERROR_LOG", `%{DATESTAMP:timestamp} \[%{DATA:err_severity}\] (%{NUMBER:pid:int}#%{NUMBER}: \*%{NUMBER}|\*%{NUMBER}) %{DATA:err_message}(?:, client: "?%{IPORHOST:client}"?)(?:, server: %{IPORHOST:server})(?:, request: "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}")?(?:, upstream: "%{DATA:upstream}")?(?:, host: "%{URIHOST:host}")?(?:, referrer: "%{URI:referrer}")?`)
+	// Nginx
+	g.AddPattern("NGINX_ERROR_DATESTAMP", `\d{4}/\d{2}/\d{2}[- ]%{TIME}`)
+	g.AddPattern("NGINX_ERROR_LOG", `%{NGINX_ERROR_DATESTAMP:timestamp} \[%{DATA:err_severity}\] (%{NUMBER:pid:int}#%{NUMBER}: \*%{NUMBER}|\*%{NUMBER}) %{DATA:message}(?:, client: "?%{IPORHOST:client}"?)(?:, server: %{IPORHOST:server})(?:, request: "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}")?(?:, upstream: "%{DATA:upstream}")?(?:, host: "%{URIHOST:host}")?(?:, referrer: "%{URI:referrer}")?`)
 }

main.go

@@ -9,13 +9,17 @@ import (
 	"sort"
 	"time"
 
+	"github.com/araddon/dateparse"
 	"github.com/getsentry/sentry-go"
 	"github.com/hpcloud/tail"
 	"github.com/vjeantet/grok"
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 
-func printMap(m map[string]string) {
+const MessageField = "message"
+const TimeStampField = "timestamp"
+
+func PrintMap(m map[string]string) {
 	keys := make([]string, 0, len(m))
 	for k := range m {
 		keys = append(keys, k)
@@ -24,8 +28,8 @@ func printMap(m map[string]string) {
 
 	for _, k := range keys {
 		fmt.Printf("%+15s: %s\n", k, m[k])
-
 	}
+	fmt.Println()
 }
 
 func IsDryRun() bool {
@@ -45,12 +49,12 @@ func InitSentry() {
 	err := sentry.Init(sentry.ClientOptions{})
 
 	if err != nil {
-		log.Fatal("Sentry initialization failed: %v\n", err)
+		log.Fatalf("Sentry initialization failed: %v\n", err)
 	}
 }
 
 func CaptureEvent(line string, values map[string]string) {
-	message := values["err_message"]
+	message := values[MessageField]
 	if message == "" {
 		message = line
 	}
@@ -75,6 +79,20 @@ func CaptureEvent(line string, values map[string]string) {
 	})
 }
 
+func ParseTimestamp(str string) int64 {
+	fallback := int64(0)
+	if str == "" {
+		return fallback
+	}
+
+	time, err := dateparse.ParseLocal(str)
+	if err != nil {
+		return fallback
+	}
+
+	return time.Unix()
+}
+
 func ProcessLine(line string, pattern string, g *grok.Grok) {
 	values, err := g.Parse(pattern, line)
 	if err != nil {
@@ -83,10 +101,14 @@ func ProcessLine(line string, pattern string, g *grok.Grok) {
 	}
 
 	if !IsDryRun() {
+		// Attempt to parse the timestamp
+		timestamp := ParseTimestamp(values[TimeStampField])
+
 		// Original log line
 		sentry.AddBreadcrumb(&sentry.Breadcrumb{
-			Message: line,
-			Level:   sentry.LevelInfo,
+			Message:   line,
+			Level:     sentry.LevelInfo,
+			Timestamp: timestamp,
 		})
 	}
 
@@ -96,8 +118,8 @@ func ProcessLine(line string, pattern string, g *grok.Grok) {
 
 	CaptureEvent(line, values)
 
-	log.Println(">>> Entry:")
-	printMap(values)
+	log.Println("Entry found:")
+	PrintMap(values)
 }
 
 func InitGrokProcessor() *grok.Grok {
@@ -157,14 +179,16 @@ func ProcessFile(filename string, pattern string) {
 		}
 	}
 
-	t, err := tail.TailFile(
+	follow := !*noFollow
+	tailFile, err := tail.TailFile(
 		filename,
 		tail.Config{
-			Follow:   !*noFollow,
+			Follow:   follow,
 			Location: &seekInfo,
+			ReOpen:   follow,
 		})
 
-	for line := range t.Lines {
+	for line := range tailFile.Lines {
 		ProcessLine(line.Text, pattern, g)
 
 	}