Merge e31113b062 into 4c2b0ec3be

Makefile

@@ -1,11 +1,13 @@
 all: deps check build
-format:
+format :
 	find . -iname "*.go" -exec gofmt -s -l -w {} \;
-check:
+check :
 	go vet ./...
-run:
+run :
 	go run cmd/HellPot/*.go
 deps:
 	go mod tidy -v
-build:
-	go build -trimpath -ldflags "-s -w -X main.version=`git tag --sort=-version:refname | head -n 1`" cmd/HellPot/*.go
+test :
+	go test -v ./...
+build :
+	go build -x -trimpath -ldflags "-s -w -X main.version=`git tag --sort=-version:refname | head -n 1`" cmd/HellPot/*.go

cmd/HellPot/HellPot_test.go

@@ -13,6 +13,7 @@ import (
 )
 
 func testMain(t *testing.T) (string, string, chan os.Signal, *config.Parameters, error) {
+	t.Setenv("HELLPOT_TEST_MODE", "true")
 	t.Helper()
 	stopChan := make(chan os.Signal, 1)
 
@@ -39,6 +40,7 @@ func TestHellPot(t *testing.T) {
 	}
 	confFile := filepath.Join(tDir, "HellPot_test.toml")
 	t.Setenv("HELLPOT_LOGGER_DIRECTORY", logDir)
+	t.Setenv("HELLPOT_LOGGER_RSYSLOG_ADDRESS", "local")
 	t.Setenv("HELLPOT_CONFIG", confFile)
 
 	resolvedConf, logFile, stopChan, realConfig, err := testMain(t)

cmd/HellPot/boot.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"errors"
-	"flag"
 	"fmt"
 	"io"
 	"os"
@@ -86,15 +85,17 @@ func readConfig(resolvedConf string) (*config.Parameters, error) {
 	var runningConfig *config.Parameters
 
 	f, err = os.Open(resolvedConf) // #nosec G304 go home gosec, you're drunk
+	defer func() {
+		if f != nil {
+			_ = f.Close()
+		}
+	}()
 	if err == nil {
 		runningConfig, setupErr = config.Setup(f)
 	}
 	switch {
 	case setupErr != nil:
 		println("failed to setup config: " + setupErr.Error())
-		if f != nil {
-			_ = f.Close()
-		}
 		err = setupErr
 	case err != nil:
 		println("failed to open config file for reading: " + err.Error())
@@ -105,26 +106,22 @@ func readConfig(resolvedConf string) (*config.Parameters, error) {
 		}
 		println("failed to create config file, cannot continue")
 		return nil, fmt.Errorf("failed to create config file: %w", err)
-	case runningConfig != nil:
-		_ = f.Close()
+	case runningConfig == nil:
+		err = errors.New("unknown failure resulting in missing configuration, cannot continue")
+		return nil, err
 	}
 
 	return runningConfig, err
 }
 
 func resolveConfig() (runningConfig *config.Parameters, usingDefaults bool, resolvedConf string, err error) {
-	setIfPresent := func(confRoot *flag.Flag) (ok bool) {
-		if confRoot != nil && confRoot.Value.String() != "" {
-			resolvedConf = confRoot.Value.String()
-			return true
-		}
-		return false
-	}
 	if config.CLIFlags != nil {
-		confRoot := config.CLIFlags.Lookup("config")
-		if !setIfPresent(confRoot) {
-			confRoot = config.CLIFlags.Lookup("c")
-			setIfPresent(confRoot)
+		conf := config.CLIFlags.Lookup("config")
+		if conf != nil && conf.Value.String() != "" {
+			resolvedConf = conf.Value.String()
+			if runningConfig, err = readConfig(resolvedConf); err != nil {
+				return runningConfig, false, resolvedConf, err
+			}
 		}
 	}
 
@@ -153,62 +150,64 @@ func resolveConfig() (runningConfig *config.Parameters, usingDefaults bool, reso
 	return runningConfig, false, resolvedConf, nil
 }
 
-func setup(stopChan chan os.Signal) (log zerolog.Logger, logFile string,
-	resolvedConf string, realConf *config.Parameters, err error) {
-
-	config.InitCLI()
-
-	var usingDefaults bool
-	var runningConfig *config.Parameters
-
-	if runningConfig, usingDefaults, resolvedConf, err = resolveConfig(); err != nil {
-		return
-	}
-
-	if runningConfig == nil {
-		err = errors.New("running configuration is nil, cannot continue")
-		return
-	}
-
-	// TODO: jesus bro r u ok
-	realConf = runningConfig
-	if usingDefaults && !realConf.UsingDefaults {
-		realConf.UsingDefaults = true
-	}
-	if realConf.UsingDefaults && !usingDefaults {
-		usingDefaults = true
-	}
-
-	//goland:noinspection GoNilness // we check for nil above
-	if log, err = logger.New(runningConfig.Logger); err != nil {
+func initLogger(runningConfig *config.Parameters) (log zerolog.Logger, logFile string, err error) {
+	if log, err = logger.New(&runningConfig.Logger); err != nil {
 		return
 	}
 
 	logFile = runningConfig.Logger.ActiveLogFileName
 
-	if usingDefaults {
-		log.Warn().Msg("using default configuration!")
-		print(red + "continuing with default configuration in ")
-		for i := defaultConfigWarningDelaySecs; i > 0; i-- {
-			print(strconv.Itoa(i))
-			for i := 0; i < 5; i++ {
-				time.Sleep(200 * time.Millisecond)
-				print(".")
-			}
-		}
-		print(reset + "\n")
-	}
-
-	if //goland:noinspection GoNilness
-	!runningConfig.Logger.NoColor {
-		extra.Banner()
-	}
-
-	signal.Notify(stopChan, syscall.SIGINT, syscall.SIGTERM)
-
-	if absResolvedConf, err := filepath.Abs(resolvedConf); err == nil {
-		resolvedConf = absResolvedConf
-	}
-
+	return
+}
+
+func setup(stopChan chan os.Signal) (log zerolog.Logger, logFile string,
+	resolvedConf string, runningConfig *config.Parameters, err error) {
+
+	config.InitCLI()
+
+	var usingDefaults bool
+
+	defer func() {
+		if runningConfig == nil && err == nil {
+			err = errors.New("running configuration is nil, cannot continue")
+			return
+		}
+		if usingDefaults && os.Getenv("HELLPOT_TEST_MODE") == "" {
+			log.Warn().Msg("using default configuration!")
+			print(red + "continuing with default configuration in ")
+			for i := defaultConfigWarningDelaySecs; i > 0; i-- {
+				print(strconv.Itoa(i))
+				for i := 0; i < 5; i++ {
+					time.Sleep(200 * time.Millisecond)
+					print(".")
+				}
+			}
+			print(reset + "\n")
+		}
+		signal.Notify(stopChan, syscall.SIGINT, syscall.SIGTERM)
+
+		if //goland:noinspection GoNilness
+		!runningConfig.Logger.NoColor {
+			extra.Banner()
+		}
+		if absResolvedConf, err := filepath.Abs(resolvedConf); err == nil {
+			resolvedConf = absResolvedConf
+		}
+	}()
+
+	switch {
+	case config.CLIFlags.Lookup("config") == nil, config.CLIFlags.Lookup("genconfig").Value.String() == "":
+		if runningConfig, usingDefaults, resolvedConf, err = resolveConfig(); err != nil {
+			return
+		}
+		log, logFile, err = initLogger(runningConfig)
+	default:
+		println("loading configuration file: " + config.CLIFlags.Lookup("config").Value.String())
+		if runningConfig, err = readConfig(config.CLIFlags.Lookup("config").Value.String()); err != nil {
+			return
+		}
+		resolvedConf = config.CLIFlags.Lookup("config").Value.String()
+		log, logFile, err = initLogger(runningConfig)
+	}
 	return
 }

internal/config/client_rules.go

@@ -8,10 +8,10 @@ import (
 
 type ClientRules struct {
 	// See: https://github.com/yunginnanet/HellPot/issues/23
-	UseragentDisallowStrings  []string         `koanf:"user_agent_disallow_strings"`
-	useragentDisallowStrBytes [][]byte         `koanf:"-"`
-	UseragentDisallowRegex    []string         `koanf:"user_agent_disallow_regex"`
-	useragentDisallowRegex    []*regexp.Regexp `koanf:"-"`
+	UseragentDisallowStrings  []string `koanf:"user_agent_disallow_strings"`
+	useragentDisallowStrBytes [][]byte
+	UseragentDisallowRegex    []string `koanf:"user_agent_disallow_regex"`
+	useragentDisallowRegex    []*regexp.Regexp
 }
 
 func NewClientRules(strs []string, regex []string) (*ClientRules, error) {

internal/config/command_line.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 	"slices"
+	"strconv"
 	"strings"
 
 	"github.com/yunginnanet/HellPot/internal/extra"
@@ -87,6 +88,30 @@ func InitCLI() {
 		println(err.Error())
 		os.Exit(2)
 	}
+
+	for defaultKey, defaultVal := range Defaults.val {
+		switch defaultVal.(type) {
+		case bool:
+			parsedBool, pErr := strconv.ParseBool(CLIFlags.Lookup(defaultKey).Value.String())
+			if pErr != nil {
+				continue
+			}
+			if parsedBool == Defaults.val[defaultKey].(bool) {
+				fl := CLIFlags.Lookup(defaultKey)
+				*fl = flag.Flag{}
+				fl = nil
+			}
+		case string:
+			if CLIFlags.Lookup(defaultKey).Value.String() == Defaults.val[defaultKey].(string) ||
+				CLIFlags.Lookup(defaultKey).Value.String() == "" {
+				fl := CLIFlags.Lookup(defaultKey)
+				*fl = flag.Flag{}
+				fl = nil
+			}
+		}
+
+	}
+
 	if os.Getenv("HELLPOT_CONFIG") != "" {
 		if err := CLIFlags.Set("config", os.Getenv("HELLPOT_CONFIG")); err != nil {
 			panic(err)

internal/config/defaults.go

@@ -54,6 +54,7 @@ var defOpts = map[string]interface{}{
 		"noconsole":           false,
 		"use_date_filename":   false,
 		"docker_logging":      false,
+		"rsyslog_address":     "",
 		"console_time_format": time.Kitchen,
 	},
 	"http": map[string]interface{}{

internal/config/defaults_test.go

@@ -21,6 +21,7 @@ func TestDefaults(t *testing.T) {
 			"http",
 			"performance",
 			"deception",
+			"bespoke",
 		} {
 			total += bytes.Count(bs, []byte(needle)) + 3 // name plus brackets and newline
 			if !bytes.Contains(bs, []byte(needle)) {

internal/config/models.go

@@ -11,11 +11,11 @@ import (
 
 // Parameters represents the configuration for HellPot.
 type Parameters struct {
-	HTTP    HTTP                  `koanf:"http"`
-	Logger  *logger.Configuration `koanf:"logger"`
-	Bespoke Customization         `koanf:"bespoke"`
-	Perf    Performance           `koanf:"performance"`
-	Liar    Deception             `koanf:"deception"`
+	HTTP    HTTP                 `koanf:"http"`
+	Logger  logger.Configuration `koanf:"logger"`
+	Bespoke Customization        `koanf:"bespoke"`
+	Perf    Performance          `koanf:"performance"`
+	Liar    Deception            `koanf:"deception"`
 
 	IdleHands DevilsPlaythings `koanf:"experimental"`
 

internal/config/setup.go

@@ -10,6 +10,8 @@ import (
 	flags "github.com/knadh/koanf/providers/basicflag"
 	"github.com/knadh/koanf/providers/env"
 	"github.com/knadh/koanf/v2"
+
+	"github.com/yunginnanet/HellPot/internal/logger"
 )
 
 type readerProvider struct {
@@ -67,17 +69,21 @@ func (p *Parameters) merge(ogk *koanf.Koanf, newk *koanf.Koanf, friendlyName str
 		return nil
 	}
 
-	for k, v := range newKeys {
-		if !ogk.Exists(k) {
-			if err := ogk.Set(k, v); err != nil {
-				panic(fmt.Sprintf("failed to set key %s: %v", k, err))
-			}
-			dirty = true
-			continue
+	valIsEmpty := func(v any) bool {
+		switch v.(type) {
+		case string:
+			return v.(string) == ""
+		case []string:
+			return len(v.([]string)) == 0
+		default:
+			return false
 		}
+	}
 
+	for k, v := range newKeys {
 		ogv := ogk.Get(k)
-		if ogv == nil {
+		if ogk.Exists(k) && valIsEmpty(ogv) && !valIsEmpty(v) {
+			println("setting newer value for key " + k + " to " + fmt.Sprintf("%v", v) + " from " + friendlyName)
 			if err := ogk.Set(k, v); err != nil {
 				panic(fmt.Sprintf("failed to set key %s: %v", k, err))
 			}
@@ -189,5 +195,16 @@ func Setup(source io.Reader) (*Parameters, error) {
 		return nil, fmt.Errorf("failed to unmarshal config: %w", err)
 	}
 
+	// for some reason the logger config pointer is not getting populated, do it manually
+	p.Logger = logger.Configuration{}
+	for k, v := range k.All() {
+		if strings.HasPrefix(k, "logger.") {
+			println("setting logger config key " + k + " to " + fmt.Sprintf("%v", v))
+			if err := p.Logger.Set(k, v); err != nil {
+				return nil, fmt.Errorf("failed to set logger config: %w", err)
+			}
+		}
+	}
+
 	return p, nil
 }

internal/config/setup_test.go

@@ -6,6 +6,7 @@ import (
 )
 
 func TestSetup(t *testing.T) {
+	t.Setenv("HELLPOT_TEST_MODE", "true")
 	t.Run("Success", SetupSuccess)
 	t.Run("NoFailureOnNilSource", SetupNoFailureOnNilSource)
 	t.Run("FailureOnReadConfig", SetupFailureOnReadConfig)
@@ -20,6 +21,10 @@ bind_addr = "5.5.5.5"
 [http.router]
 catchall = true
 makerobots = false
+
+[logger]
+debug = true
+rsyslog_address = "local"
 `)
 
 	params, err := Setup(source)
@@ -49,7 +54,24 @@ makerobots = false
 	if params.HTTP.Router.CatchAll != true {
 		t.Errorf("Expected true, got %v", params.HTTP.Router.CatchAll)
 	}
-
+	if params.source.Get("http.router.makerobots") != false {
+		t.Errorf("Expected false, got %v", params.source.Get("http.router.makerobots"))
+	}
+	if params.HTTP.Router.MakeRobots != false {
+		t.Errorf("Expected false, got %v", params.HTTP.Router.MakeRobots)
+	}
+	if params.source.Get("logger.debug") != true {
+		t.Errorf("Expected true, got %v", params.source.Get("logger.debug"))
+	}
+	if params.Logger.Debug != true {
+		t.Errorf("Expected true, got %v", params.Logger.Debug)
+	}
+	if params.source.Get("logger.rsyslog_address") != "local" {
+		t.Errorf("Expected local, got %v", params.source.Get("logger.rsyslog_address"))
+	}
+	if params.Logger.RSyslog != "local" {
+		t.Errorf("Expected local, got %v", params.Logger.RSyslog)
+	}
 }
 
 func SetupNoFailureOnNilSource(t *testing.T) {

internal/logger/logger.go

@@ -7,6 +7,8 @@ import (
 	"log/syslog"
 	"os"
 	"path/filepath"
+	"reflect"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -23,20 +25,90 @@ type Configuration struct {
 	DockerLogging bool `koanf:"docker_logging"`
 	// ConsoleTimeFormat sets the time format for the console.
 	// The string is passed to time.Format() down the line.
-	ConsoleTimeFormat string
+	ConsoleTimeFormat string `koanf:"console_time_format"`
 
 	// TimeDateFilename sets the log file name to include the date and time.
 	TimeDateFilename bool `koanf:"use_date_filename"`
 
-	Directory string `koanf:"directory"`
-	File      string `koanf:"log_file"`
-	RSyslog   string `koanf:"rsyslog_address"`
+	Directory     string `koanf:"directory"`
+	File          string `koanf:"log_file"`
+	RSyslog       string `koanf:"rsyslog_address"`
+	rsyslogTarget string
 
 	ActiveLogFileName string `koanf:"active_log_file_name"`
 
 	Outputs []io.Writer `koanf:"-"`
 }
 
+func (c *Configuration) Set(k string, v any) error {
+	k = strings.ToLower(k)
+	k = strings.TrimPrefix(k, "logger.")
+	k = strings.ReplaceAll(k, "__", "_")
+	k = strings.ReplaceAll(k, ".", "_")
+
+	ref := reflect.ValueOf(c)
+	if ref.Kind() != reflect.Ptr {
+		panic("not a pointer")
+	}
+	ref = ref.Elem()
+	if ref.Kind() != reflect.Struct {
+		panic("not a struct")
+	}
+
+	var field reflect.Value
+
+	for i := 0; i < ref.NumField(); i++ {
+		strutTag := ref.Type().Field(i).Tag.Get("koanf")
+		if strings.ToLower(strutTag) == strings.ToLower(k) {
+			field = ref.Field(i)
+			break
+		}
+	}
+
+	if field == (reflect.Value{}) {
+		return fmt.Errorf("field %s does not exist", k)
+	}
+
+	if !field.CanSet() {
+		return fmt.Errorf("field %s cannot be set", k)
+	}
+
+	switch field.Kind() {
+	case reflect.Bool:
+		if vstr, vstrok := v.(string); vstrok {
+			if vb, err := strconv.ParseBool(vstr); err == nil {
+				field.SetBool(vb)
+				return nil
+			}
+		}
+		if b, ok := v.(bool); ok {
+			field.SetBool(b)
+			return nil
+		}
+		return fmt.Errorf("field %s is not a bool", k)
+	case reflect.String:
+		if s, ok := v.(string); ok {
+			field.SetString(s)
+			return nil
+		}
+		return fmt.Errorf("field %s is not a string", k)
+	case reflect.Slice:
+		if s, ok := v.([]string); ok {
+			field.Set(reflect.ValueOf(s))
+			return nil
+		}
+		return fmt.Errorf("field %s is not a slice", k)
+	case reflect.Int:
+		if i, ok := v.(int); ok {
+			field.SetInt(int64(i))
+			return nil
+		}
+		return fmt.Errorf("field %s is not an int", k)
+	default:
+		return fmt.Errorf("field %s is not a supported type (%T)", k, v)
+	}
+}
+
 func (c *Configuration) findFallbackDir() error {
 	locs := []string{"/var/log"}
 	uconf, err := os.UserHomeDir()
@@ -143,7 +215,10 @@ func (c *Configuration) setupDirAndFile() error {
 	return nil
 }
 
-func (c *Configuration) setupSyslog() error {
+func (c *Configuration) setupSyslog() (error, bool) {
+	if c.RSyslog == "" {
+		return nil, false
+	}
 	var (
 		err   error
 		proto string
@@ -151,6 +226,9 @@ func (c *Configuration) setupSyslog() error {
 		conn  *syslog.Writer
 	)
 	switch {
+	case strings.ToLower(c.RSyslog) == "local":
+		proto = ""
+		addr = ""
 	case strings.Contains(c.RSyslog, "://"):
 		proto = strings.Split(c.RSyslog, "://")[0]
 		addr = strings.Split(c.RSyslog, "://")[1]
@@ -161,24 +239,37 @@ func (c *Configuration) setupSyslog() error {
 		proto = "udp"
 		addr = c.RSyslog + ":514"
 	}
-	if conn, err = syslog.Dial(proto, addr, syslog.LOG_INFO, "HellPot"); err != nil {
-		return fmt.Errorf("failed to dial syslog server: %w", err)
+
+	if proto != "" && addr != "" {
+		println("dialing syslog server: " + c.rsyslogTarget + "...")
 	}
 
+	syslogLogLevel := syslog.LOG_INFO
+	if c.Debug || c.Trace {
+		syslogLogLevel = syslog.LOG_DEBUG
+	}
+
+	if conn, err = syslog.Dial(proto, addr, syslogLogLevel, "HellPot"); err != nil {
+		return fmt.Errorf("failed to dial syslog server: %w", err), false
+	}
+
+	c.rsyslogTarget = proto + "://" + addr
+
 	c.Outputs = append(c.Outputs, zerolog.SyslogLevelWriter(conn))
 
-	return nil
+	return nil, true
 }
 
-func (c *Configuration) SetupOutputs() error {
+func (c *Configuration) SetupOutputs() (err error, rsyslogEnabled bool) {
 	if c.Directory != "" || c.File != "" {
-		if err := c.setupDirAndFile(); err != nil {
-			return fmt.Errorf("failed to setup log file: %w", err)
+		if err = c.setupDirAndFile(); err != nil {
+			return fmt.Errorf("failed to setup log file: %w", err), false
 		}
 	}
+
 	if c.RSyslog != "" {
-		if err := c.setupSyslog(); err != nil {
-			return fmt.Errorf("failed to setup syslog: %w", err)
+		if err, rsyslogEnabled = c.setupSyslog(); err != nil {
+			return fmt.Errorf("failed to setup syslog: %w", err), false
 		}
 	}
 
@@ -186,7 +277,7 @@ func (c *Configuration) SetupOutputs() error {
 
 	for _, out := range c.Outputs {
 		if out == nil {
-			return fmt.Errorf("nil output provided")
+			return fmt.Errorf("nil output provided"), false
 		}
 		if out == os.Stdout || out == os.Stderr {
 			consoleSeen = true
@@ -197,7 +288,7 @@ func (c *Configuration) SetupOutputs() error {
 		c.Outputs = append(c.Outputs, os.Stdout)
 	}
 
-	return nil
+	return nil, rsyslogEnabled
 }
 
 var once = &sync.Once{}
@@ -228,7 +319,9 @@ func New(conf *Configuration) (zerolog.Logger, error) {
 	if err := conf.Validate(); err != nil {
 		return zerolog.Logger{}, fmt.Errorf("invalid logger configuration: %w", err)
 	}
-	if err := conf.SetupOutputs(); err != nil {
+	var err error
+	var rsyslogEnabled bool
+	if err, rsyslogEnabled = conf.SetupOutputs(); err != nil {
 		return zerolog.Logger{}, fmt.Errorf("failed to setup logger outputs: %w", err)
 	}
 	for i, output := range conf.Outputs {
@@ -246,5 +339,10 @@ func New(conf *Configuration) (zerolog.Logger, error) {
 	if conf.Trace {
 		_log = _log.Level(zerolog.TraceLevel)
 	}
+
+	if rsyslogEnabled {
+		_log.Info().Str("target", conf.rsyslogTarget).Msg("remote syslog connection established")
+	}
+
 	return _log, nil
 }