随笔档案 (761)

How to change the cnfiguration for rate limiting in flight?

How to change the configuration of pgxpool without restarting the application?

How to dynamically reload the configuration of SMTP when the username or password is changed?

I use Viper to watch and reload the config changes, and applying the changes at runtime using wrappers.

Need to create a Viper instance for each config file.

zzh@ZZHPC:/zdata/Github/greenlight$ tree
.
├── bin
├── cmd
│   └── api
│       ├── errors.go
│       ├── healthcheck.go
│       ├── helpers.go
│       ├── main.go
│       ├── middleware.go
│       ├── movie.go
│       ├── routes.go
│       ├── server.go
│       └── user.go
├── create_db.sql
├── go.mod
├── go.sum
├── internal
│   ├── config
│   │   ├── config.go
│   │   ├── dynamic_db_secret.env
│   │   ├── dynamic.env
│   │   └── dynamic_smtp_secret.env
│   ├── data
│   │   ├── db.go
│   │   ├── filter.go
│   │   ├── models.go
│   │   ├── movie.go
│   │   ├── runtime.go
│   │   └── user.go
│   ├── mail
│   │   ├── sender.go
│   │   └── templates
│   │       └── user_welcome.html
│   └── validator
│       └── validator.go
├── Makefile
├── migrations
│   ├── 000001_create_movie_table.down.sql
│   ├── 000001_create_movie_table.up.sql
│   ├── 000002_add_movie_check_constraints.down.sql
│   ├── 000002_add_movie_check_constraints.up.sql
│   ├── 000003_add_movie_indexes.down.sql
│   ├── 000003_add_movie_indexes.up.sql
│   ├── 000004_create_users_table.down.sql
│   └── 000004_create_users_table.up.sql
├── README.md
└── remote

config.go:

package config

import (
    "time"

    "github.com/spf13/viper"
)

// Config stores configuration that can be dynamically reloaded at runtime.
type Config struct {
    DBUsername            string        `mapstructure:"DB_USERNAME"`
    DBPassword            string        `mapstructure:"DB_PASSWORD"`
    DBServer              string        `mapstructure:"DB_SERVER"`
    DBPort                int           `mapstructure:"DB_PORT"`
    DBName                string        `mapstructure:"DB_NAME"`
    DBSSLMode             string        `mapstructure:"DB_SSLMODE"`
    DBPoolMaxConns        int           `mapstructure:"DB_POOL_MAX_CONNS"`
    DBPoolMaxConnIdleTime time.Duration `mapstructure:"DB_POOL_MAX_CONN_IDLE_TIME"`
    LimiterRps            float64       `mapstructure:"LIMITER_RPS"`
    LimiterBurst          int           `mapstructure:"LIMITER_BURST"`
    LimiterEnabled        bool          `mapstructure:"LIMITER_ENABLED"`
    SMTPUsername          string        `mapstructure:"SMTP_USERNAME"`
    SMTPPassword          string        `mapstructure:"SMTP_PASSWORD"`
    SMTPAuthAddress       string        `mapstructure:"SMTP_AUTH_ADDRESS"`
    SMTPServerAddress     string        `mapstructure:"SMTP_SERVER_ADDRESS"`
    LoadTime              time.Time
}

// LimiterConfig stores configuration for rate limiting.
type LimiterConfig struct {
    Rps     float64
    Burst   int
    Enabled bool
}

// SMTPConfig stores configuration for sending emails.
type SMTPConfig struct {
    Username      string
    Password      string
    AuthAddress   string
    ServerAddress string
}

// LoadConfig loads configuration from a config file to a Config instance.
func LoadConfig(v *viper.Viper, cfgPath, cfgType, cfgName string, cfg *Config) error {
    v.AddConfigPath(cfgPath)
    v.SetConfigType(cfgType)
    v.SetConfigName(cfgName)

    err := v.ReadInConfig()
    if err != nil {
        return err
    }

    err = v.Unmarshal(cfg)
    if err != nil {
        return err
    }

    cfg.LoadTime = time.Now()

    return nil
}

db.go:

package data

import (
    "context"
    "time"

    "github.com/jackc/pgx/v5/pgxpool"
)

// PoolWrapper wraps a *pgxpool.Pool.
type PoolWrapper struct {
    Pool *pgxpool.Pool
}

// CreatePool creates a *pgxpool.Pool and assigns it to the wrapper's Pool field.
func (pw *PoolWrapper) CreatePool(connString string) error {
    ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
    defer cancel()

    p, err := pgxpool.New(ctx, connString)
    if err != nil {
        return err
    }

    err = p.Ping(ctx)
    if err != nil {
        p.Close()
        return err
    }

    pw.Pool = p

    return nil
}

models.go:

package data

import (
    "errors"
)

var (
    ErrMsgViolateUniqueConstraint = "duplicate key value violates unique constraint"

    ErrRecordNotFound = errors.New("record not found")
    ErrEditConflict   = errors.New("edit conflict")
)

// Models puts models together in one struct.
type Models struct {
    Movie MovieModel
    User  UserModel
}

// NewModels returns a Models struct containing the initialized models.
func NewModels(pw *PoolWrapper) Models {
    return Models{
        Movie: MovieModel{DB: pw},
        User:  UserModel{DB: pw},
    }
}

user.go

...

// User represents an individual user.
type User struct {
    ID        int64     `json:"id"`
    CreatedAt time.Time `json:"created_at"`
    Name      string    `json:"name"`
    Email     string    `json:"email"`
    Password  password  `json:"-"`
    Activated bool      `json:"activated"`
    Version   int       `json:"version"`
}

...

// UserModel struct wraps a database connection pool wrapper.
type UserModel struct {
    DB *PoolWrapper
}

// Insert inserts a new record in the users table.
func (m UserModel) Insert(user *User) error {
    query := `INSERT INTO users (name, email, password_hash, activated) 
              VALUES ($1, $2, $3, $4) 
              RETURNING id, created_at, version`

    args := []any{user.Name, user.Email, user.Password.hash, user.Activated}

    ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
    defer cancel()

    err := m.DB.Pool.QueryRow(ctx, query, args...).Scan(&user.ID, &user.CreatedAt, &user.Version)
    if err != nil {
        switch {
        case strings.Contains(err.Error(), ErrMsgViolateUniqueConstraint) && strings.Contains(err.Error(), "email"):
            return ErrDuplicateEmail
        default:
            return err
        }
    }

    return nil
}

...

sender.go:

package mail

import (
    "bytes"
    "embed"
    "html/template"
    "net/smtp"

    "github.com/jordan-wright/email"
    "greenlight.zzh.net/internal/config"
)

//go:embed "templates"
var templateFS embed.FS

// EmailSender wraps a *config.SMTPConfig which stores configuration for sending emails.
type EmailSender struct {
    SMTPCfg *config.SMTPConfig
}

// Send sends an email whose subject and content are read from a template file.
// Use a pointer receiver because the fields of EmailSender can be dynamically loaded.
func (sender *EmailSender) Send(from, to, templateFile string, data any) error {
    tmpl, err := template.New("email").ParseFS(templateFS, "templates/"+templateFile)
    if err != nil {
        return err
    }

    // Execute the named tempalte "subject", passing in the dynamic data and storing the 
    // result in a bytes.Buffer variable.
    subject := new(bytes.Buffer)
    err = tmpl.ExecuteTemplate(subject, "subject", data)
    if err != nil {
        return err
    }

    // Execute the named tempalte "plainBody", passing in the dynamic data and storing the 
    // result in a bytes.Buffer variable.
    plainBody := new(bytes.Buffer)
    err = tmpl.ExecuteTemplate(plainBody, "plainBody", data)
    if err != nil {
        return err
    }

    htmlBody := new(bytes.Buffer)
    err = tmpl.ExecuteTemplate(htmlBody, "htmlBody", data)
    if err != nil {
        return err
    }

    e := email.NewEmail()
    e.From = from
    e.To = []string{to}
    e.Subject = subject.String()
    e.Text = plainBody.Bytes()
    e.HTML = htmlBody.Bytes()

    smtpAuth := smtp.PlainAuth("", sender.SMTPCfg.Username, sender.SMTPCfg.Password, sender.SMTPCfg.AuthAddress)
    return e.Send(sender.SMTPCfg.ServerAddress, smtpAuth)
}

main.go:

package main

import (
    "flag"
    "fmt"
    "log/slog"
    "os"
    "time"

    "github.com/fsnotify/fsnotify"
    "github.com/spf13/viper"
    "greenlight.zzh.net/internal/config"
    "greenlight.zzh.net/internal/data"
    "greenlight.zzh.net/internal/mail"
)

// Declare a string containing the application version number. Later in the book we'll
// generate this automatically at build time, but for now we'll just store the version
// number as a hard-coded global constant.
const version = "1.0.0"

type appConfig struct {
    serverAddress string
    env           string
    dbConnString  string
    limiter       *config.LimiterConfig
    smtp          *config.SMTPConfig
}

// Define an application struct to hold the dependencies for our HTTP handlers, helpers,
// and middleware.
type application struct {
    config      appConfig
    logger      *slog.Logger
    models      data.Models
    emailSender *mail.EmailSender
}

func main() {
    var (
        configPath    string
        serverAddress string
        env           string
    )

    // Read the location of config files for dynamic configuration from command line.
    flag.StringVar(&configPath, "config-path", "internal/config", "The directory that contains configuration files.")

    // Read static configuration from command line.
    flag.StringVar(&serverAddress, "server-address", ":4000", "The server address of this application.")
    flag.StringVar(&env, "env", "development", "Environment (development|staging|production)")

    // Parse command line parameters.
    flag.Parse()

    logger := slog.New(slog.NewTextHandler(os.Stdout, nil))

    var cfgDynamic config.Config

    // Load dynamic configuration.
    viperDynamic := viper.New()
    err := config.LoadConfig(viperDynamic, configPath, "env", "dynamic", &cfgDynamic)
    if err != nil {
        logger.Error(err.Error())
        os.Exit(1)
    }

    // Load dynamic DB configuration.
    viperDynamicDB := viper.New()
    err = config.LoadConfig(viperDynamicDB, configPath, "env", "dynamic_db_secret", &cfgDynamic)
    if err != nil {
        logger.Error(err.Error())
        os.Exit(1)
    }

    // Load dynamic SMTP configuration.
    viperDynamicSMTP := viper.New()
    err = config.LoadConfig(viperDynamicSMTP, configPath, "env", "dynamic_smtp_secret", &cfgDynamic)
    if err != nil {
        logger.Error(err.Error())
        os.Exit(1)
    }

    // Create an appConfig instance.
    cfg := appConfig{
        serverAddress: serverAddress,
        env:           env,
        dbConnString: fmt.Sprintf(
            "postgres://%s:%s@%s:%d/%s?sslmode=%s&pool_max_conns=%d&pool_max_conn_idle_time=%s",
            cfgDynamic.DBUsername, cfgDynamic.DBPassword, cfgDynamic.DBServer, cfgDynamic.DBPort, cfgDynamic.DBName,
            cfgDynamic.DBSSLMode, cfgDynamic.DBPoolMaxConns, cfgDynamic.DBPoolMaxConnIdleTime,
        ),
        limiter: &config.LimiterConfig{
            Rps:     cfgDynamic.LimiterRps,
            Burst:   cfgDynamic.LimiterBurst,
            Enabled: cfgDynamic.LimiterEnabled,
        },
        smtp: &config.SMTPConfig{
            Username:      cfgDynamic.SMTPUsername,
            Password:      cfgDynamic.SMTPPassword,
            AuthAddress:   cfgDynamic.SMTPAuthAddress,
            ServerAddress: cfgDynamic.SMTPServerAddress,
        },
    }

    // Create a database connection pool wrapper.
    var poolWrapper data.PoolWrapper
    err = poolWrapper.CreatePool(cfg.dbConnString)
    if err != nil {
        logger.Error(err.Error())
        os.Exit(1)
    }
    defer poolWrapper.Pool.Close()
    logger.Info("database connection pool established")

    // Create the application instance.
    app := &application{
        config:      cfg,
        logger:      logger,
        models:      data.NewModels(&poolWrapper),
        emailSender: &mail.EmailSender{SMTPCfg: cfg.smtp},
    }

    // Watch and reload dynamic.env config file.
    go func() {
        viperDynamic.OnConfigChange(func(in fsnotify.Event) {
            // A change in the config file can cause two 'write' events.
            // Only need to respond once. We respond to the first one.
            if time.Since(cfgDynamic.LoadTime) > time.Duration(100*time.Millisecond) {
                logger.Info("configuration change detected", "filename", in.Name, "operation", in.Op)

                // Reload the config file if any change is detected.
                err := config.LoadConfig(viperDynamic, configPath, "env", "dynamic", &cfgDynamic)
                if err != nil {
                    logger.Error(err.Error())
                    os.Exit(1)
                }

                cfg.limiter.Rps = cfgDynamic.LimiterRps
                cfg.limiter.Burst = cfgDynamic.LimiterBurst
                cfg.limiter.Enabled = cfgDynamic.LimiterEnabled
            }
        })
        viperDynamic.WatchConfig()
    }()

    // Watch and reload dynamic_db_secret.env config file.
    go func() {
        viperDynamicDB.OnConfigChange(func(in fsnotify.Event) {
            if time.Since(cfgDynamic.LoadTime) > time.Duration(100*time.Millisecond) {
                logger.Info("configuration change detected", "filename", in.Name, "operation", in.Op)

                err := config.LoadConfig(viperDynamicDB, configPath, "env", "dynamic_db_secret", &cfgDynamic)
                if err != nil {
                    logger.Error(err.Error())
                    os.Exit(1)
                }

                cfg.dbConnString = fmt.Sprintf(
                    "postgres://%s:%s@%s:%d/%s?sslmode=%s&pool_max_conns=%d&pool_max_conn_idle_time=%s",
                    cfgDynamic.DBUsername, cfgDynamic.DBPassword, cfgDynamic.DBServer, cfgDynamic.DBPort, cfgDynamic.DBName,
                    cfgDynamic.DBSSLMode, cfgDynamic.DBPoolMaxConns, cfgDynamic.DBPoolMaxConnIdleTime,
                )

                // Close the old database connection pool and create a new one.
                poolWrapper.Pool.Close()
                err = poolWrapper.CreatePool(cfg.dbConnString)
                if err != nil {
                    logger.Error(err.Error())
                    os.Exit(1)
                }
            }
        })
        viperDynamicDB.WatchConfig()
    }()

    // Watch and reload dynamic_smtp_secret.env config file.
    go func() {
        viperDynamicSMTP.OnConfigChange(func(in fsnotify.Event) {
            if time.Since(cfgDynamic.LoadTime) > time.Duration(100*time.Millisecond) {
                logger.Info("configuration change detected", "filename", in.Name, "operation", in.Op)

                err := config.LoadConfig(viperDynamicSMTP, configPath, "env", "dynamic_smtp_secret", &cfgDynamic)
                if err != nil {
                    logger.Error(err.Error())
                    os.Exit(1)
                }

                cfg.smtp.Username = cfgDynamic.SMTPUsername
                cfg.smtp.Password = cfgDynamic.SMTPPassword
                cfg.smtp.AuthAddress = cfgDynamic.SMTPAuthAddress
                cfg.smtp.ServerAddress = cfgDynamic.SMTPServerAddress
            }
        })
        viperDynamicSMTP.WatchConfig()
    }()

    err = app.serve()
    if err != nil {
        logger.Error(err.Error())
        os.Exit(1)
    }
}