Refactor (#2)

* handle errors

* use DialContext

* refactor dev env

* server: use context for http request

* server: refactor

* server: create Agent client when needed

* server: rename

* server: rename

* server: log

* server: return err

* server: do request with context

* server: rename

* server: do sequential requests to mysql and proxy

* sever: define rawprocess

* server: get raw processes inside a function

* server: rename

* server: log

* add detail info

* server: add cant connect case

* server: omitempty

* server: re-order the table

* server: make detail a struct

* server: define agent address early

* server: save multiple details

* server: create kimo processes inside server file

* static: sort by id

* server: convert combine method to function

* config: delete unused configs

* server: set metrics after process generation

* server: consider tcpproxy absence

* ui: group columns

* server: log

* server: use context

* agent: move polling logic to poll.go

* agent: use context

* agent: convert method to func

* server: remove force param

* server: do polling in another file

* server: define context above

* server: use context

* use buffered channels

* agent: check only tcp connections

* server: simplify metrics

* server: format cmdline

* use yaml config

* pass only necessary config

* delete unused flag

* agent returns cmdline as string for the sake of simplicity

* server: anonymize unknown cmdline

* agent: delete unused field

* show connection status

* handle agent errors

* server: check tcp proxy existence

* server: pass IPPort to agent client

* agent: rename

* use mutex

* server: rename

* server: rename

* server: define getters for simplicity

* doc & organize

* metric: rename

* agent: create agent.go file to organize better

* agent: organize

Author: muraty

Dockerfile-kimo

@@ -1,18 +1,33 @@
 FROM golang:1.23
 
-RUN apt-get update && apt-get -y install default-mysql-client
+# Install system dependencies
+RUN apt-get update && \
+    apt-get -y install default-mysql-client && \
+    rm -rf /var/lib/apt/lists/*
 
 ENV GOPATH=/go
 ENV PATH=$PATH:$GOPATH/bin
 
-RUN mkdir /go/src/kimo
-COPY go.mod go.sum /go/src/kimo/
-COPY ./server/static/ /go/src/kimo/server/static
+WORKDIR /go/src/kimo
 
-RUN cd /go/src/kimo && go install github.com/rakyll/statik
-RUN cd /go/src/kimo && /go/bin/statik -src=./server/static -include='*.html'
+# Copy dependency files first to leverage cache
+COPY go.mod go.sum ./
 
-COPY . /go/src/kimo
-RUN cd /go/src/kimo && go install
+# Download dependencies separately
+RUN go mod download
 
-ADD config.toml /etc/kimo.toml
+# Install statik before copying other files
+RUN go install github.com/rakyll/statik
+
+# Copy only static files needed for statik
+COPY ./server/static/ ./server/static/
+RUN /go/bin/statik -src=./server/static -include='*.html'
+
+# Copy remaining source code
+COPY . .
+
+# Build the application
+RUN go build -o /go/bin/kimo
+
+# Add config file
+COPY config.yaml /etc/kimo.yaml

Dockerfile-tcpproxy

@@ -7,15 +7,15 @@ build:
 	go get github.com/rakyll/statik
 	$(GOPATH)/bin/statik -src=./server/static -include='*.html'
 	go install
-up:
+build-dependencies:
 	docker-compose stop
 	docker-compose rm -fsv
 	docker-compose build mysql
 	docker-compose build kimo
-	docker-compose build kimo-agent
-	docker-compose build kimo-server
-	docker-compose build tcpproxy
-	docker-compose up --scale kimo-agent=5
-
+up:
+	docker-compose stop
+	docker-compose rm -fsv
+	docker-compose build kimo
+	docker-compose up --build kimo-server kimo-agent
 lint:
 	golangci-lint run

Makefile

@@ -0,0 +1,68 @@
+package agent
+
+import (
+	"context"
+	"kimo/config"
+	"net/http"
+	"os"
+	"sync"
+
+	"github.com/cenkalti/log"
+	gopsutilNet "github.com/shirou/gopsutil/v4/net"
+)
+
+// Agent is type for handling agent operations
+type Agent struct {
+	Config   *config.AgentConfig
+	conns    []gopsutilNet.ConnectionStat
+	Hostname string
+	mu       sync.RWMutex // protects conns
+}
+
+// NewAgent creates an returns a new Agent
+func NewAgent(cfg *config.AgentConfig) *Agent {
+	d := new(Agent)
+	d.Config = cfg
+	d.Hostname = getHostname()
+	return d
+}
+
+// SetConns sets connections with lock.
+func (a *Agent) SetConns(conns []gopsutilNet.ConnectionStat) {
+	a.mu.Lock()
+	a.conns = conns
+	a.mu.Unlock()
+}
+
+// GetConns gets connections with lock.
+func (a *Agent) GetConns() []gopsutilNet.ConnectionStat {
+	a.mu.RLock()
+	defer a.mu.RUnlock()
+	return a.conns
+}
+
+// getHostname returns hostname.
+func getHostname() string {
+	hostname, err := os.Hostname()
+	if err != nil {
+		log.Errorf("Hostname could not found")
+		hostname = "UNKNOWN"
+	}
+	return hostname
+}
+
+// Run starts the http server and begins listening for HTTP requests.
+func (a *Agent) Run() error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	go a.pollConns(ctx)
+
+	http.HandleFunc("/proc", a.Process)
+	err := http.ListenAndServe(a.Config.ListenAddress, nil)
+	if err != nil {
+		log.Errorln(err.Error())
+		return err
+	}
+	return nil
+}

agent/agent.go

@@ -2,42 +2,29 @@ package agent
 
 import (
 	"encoding/json"
-	"kimo/config"
-	"kimo/types"
 	"net/http"
-	"os"
 	"strconv"
-	"time"
 
 	"github.com/cenkalti/log"
 	gopsutilNet "github.com/shirou/gopsutil/v4/net"
 	gopsutilProcess "github.com/shirou/gopsutil/v4/process"
 )
 
-// Agent is type for handling agent operations
-type Agent struct {
-	Config   *config.Agent
-	Conns    []gopsutilNet.ConnectionStat
-	Hostname string
+// Response contains basic process information for API responses.
+type Response struct {
+	Status  string `json:"status"`
+	Pid     int32  `json:"pid"`
+	Name    string `json:"name"`
+	CmdLine string `json:"cmdline"`
 }
 
-// NewAgent is constuctor function for Agent type
-func NewAgent(cfg *config.Config) *Agent {
-	d := new(Agent)
-	d.Config = &cfg.Agent
-	d.Hostname = getHostname()
-	return d
-}
-
-func getHostname() string {
-	hostname, err := os.Hostname()
-	if err != nil {
-		log.Errorf("Hostname could not found")
-		hostname = "UNKNOWN"
-	}
-	return hostname
+// NetworkProcess represents process with its network connection.
+type NetworkProcess struct {
+	process *gopsutilProcess.Process
+	conn    gopsutilNet.ConnectionStat
 }
 
+// parsePortParam parses and returns port number from the request.
 func parsePortParam(w http.ResponseWriter, req *http.Request) (uint32, error) {
 	portParam, ok := req.URL.Query()["port"]
 	log.Debugf("Looking for process of port: %s\n", portParam)
@@ -55,13 +42,9 @@ func parsePortParam(w http.ResponseWriter, req *http.Request) (uint32, error) {
 	return uint32(p), nil
 }
 
-type hostProc struct {
-	process *gopsutilProcess.Process
-	conn    gopsutilNet.ConnectionStat
-}
-
-func (a *Agent) findProc(port uint32) *hostProc {
-	for _, conn := range a.Conns {
+// findProcess finds process from connections by given port.
+func findProcess(port uint32, conns []gopsutilNet.ConnectionStat) *NetworkProcess {
+	for _, conn := range conns {
 		if conn.Laddr.Port != port {
 			continue
 		}
@@ -77,93 +60,55 @@ func (a *Agent) findProc(port uint32) *hostProc {
 			return nil
 		}
 
-		return &hostProc{
+		return &NetworkProcess{
 			process: process,
 			conn:    conn,
 		}
 	}
 	return nil
 }
 
-func (a *Agent) createAgentProcess(proc *hostProc) *types.AgentProcess {
-	if proc == nil {
+// createResponse creates Response from given NetworkProcess parameter.
+func createResponse(np *NetworkProcess) *Response {
+	if np == nil {
 		return nil
 	}
-	name, err := proc.process.Name()
+	name, err := np.process.Name()
 	if err != nil {
 		name = ""
 	}
-	cl, err := proc.process.CmdlineSlice()
+	cmdline, err := np.process.Cmdline()
 	if err != nil {
-		log.Debugf("Cmdline could not found for %d\n", proc.process.Pid)
+		log.Debugf("Cmdline could not found for %d\n", np.process.Pid)
 	}
-	return &types.AgentProcess{
-		Laddr:    types.IPPort{IP: proc.conn.Laddr.IP, Port: proc.conn.Laddr.Port},
-		Status:   proc.conn.Status,
-		Pid:      proc.conn.Pid,
-		Name:     name,
-		CmdLine:  cl,
-		Hostname: a.Hostname,
+	return &Response{
+		Status:  np.conn.Status,
+		Pid:     np.conn.Pid,
+		Name:    name,
+		CmdLine: cmdline,
 	}
 }
 
 // Process is handler for serving process info
 func (a *Agent) Process(w http.ResponseWriter, req *http.Request) {
+	w.Header().Set("X-Kimo-Hostname", a.Hostname)
+
 	// todo: cache result for a short period (10s? 30s?)
 	port, err := parsePortParam(w, req)
 	if err != nil {
 		http.Error(w, "port param is required", http.StatusBadRequest)
 		return
 	}
-	p := a.findProc(port)
-	ap := a.createAgentProcess(p)
-
-	w.Header().Set("Content-Type", "application/json")
-	if ap == nil {
-		json.NewEncoder(w).Encode(&types.AgentProcess{
-			Hostname: a.Hostname,
-		})
+	p := findProcess(port, a.GetConns())
+	if p == nil {
+		http.Error(w, "Connection not found", http.StatusNotFound)
 		return
 	}
-	json.NewEncoder(w).Encode(&ap)
-	return
-}
 
-func (a *Agent) pollConns() {
-	// todo: run with context
-	log.Debugln("Polling...")
-	ticker := time.NewTicker(a.Config.PollDuration * time.Second)
-
-	for {
-		a.getConns() // poll immediately at the initialization
-		select {
-		// todo: add return case
-		case <-ticker.C:
-			a.getConns()
-		}
-	}
-
-}
-func (a *Agent) getConns() {
-	// This is an expensive operation.
-	// So, we need to call it infrequent to prevent high load on servers those run kimo agents.
-	conns, err := gopsutilNet.Connections("all")
-	if err != nil {
-		log.Errorln(err.Error())
-		return
-	}
-	a.Conns = conns
-}
-
-// Run is main function to run http server
-func (a *Agent) Run() error {
-	go a.pollConns()
-
-	http.HandleFunc("/proc", a.Process)
-	err := http.ListenAndServe(a.Config.ListenAddress, nil)
+	w.Header().Set("Content-Type", "application/json")
+	ap := createResponse(p)
+	err = json.NewEncoder(w).Encode(&ap)
 	if err != nil {
-		log.Errorln(err.Error())
-		return err
+		http.Error(w, "Can not encode agent process", http.StatusInternalServerError)
 	}
-	return nil
 }