Debugging prod failures

2026-03-28 17:42:37 +01:00 · 2025-05-04 10:15:42 -07:00
parent 92c08c0d6b
commit 3ce8889786
10 changed files with 282 additions and 54 deletions
--- a/8
+++ b/8
@@ -113,6 +113,14 @@ ENV MESHSTREAM_MQTT_TOPIC_PREFIX=msh/US/bayarea
 ENV MESHSTREAM_MQTT_CLIENT_ID=meshstream
 ENV MESHSTREAM_CHANNEL_KEYS=""

+# MQTT connection tuning parameters
+ENV MESHSTREAM_MQTT_KEEPALIVE=60
+ENV MESHSTREAM_MQTT_CONNECT_TIMEOUT=30s
+ENV MESHSTREAM_MQTT_PING_TIMEOUT=10s
+ENV MESHSTREAM_MQTT_MAX_RECONNECT=5m
+ENV MESHSTREAM_MQTT_USE_TLS=false
+ENV MESHSTREAM_MQTT_TLS_PORT=8883
+
 # Note: Web app configuration is set at build time
 # and baked into the static files

--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -25,6 +25,15 @@ services:
      - MESHSTREAM_MQTT_USERNAME=${MESHSTREAM_MQTT_USERNAME:-meshdev}
      - MESHSTREAM_MQTT_PASSWORD=${MESHSTREAM_MQTT_PASSWORD:-large4cats}
      - MESHSTREAM_MQTT_TOPIC_PREFIX=${MESHSTREAM_MQTT_TOPIC_PREFIX:-msh/US/bayarea}
+      - MESHSTREAM_MQTT_CLIENT_ID=${MESHSTREAM_MQTT_CLIENT_ID:-meshstream-${HOSTNAME:-local}}
+      
+      # MQTT connection tuning parameters
+      - MESHSTREAM_MQTT_KEEPALIVE=${MESHSTREAM_MQTT_KEEPALIVE:-60}
+      - MESHSTREAM_MQTT_CONNECT_TIMEOUT=${MESHSTREAM_MQTT_CONNECT_TIMEOUT:-30s}
+      - MESHSTREAM_MQTT_PING_TIMEOUT=${MESHSTREAM_MQTT_PING_TIMEOUT:-10s}
+      - MESHSTREAM_MQTT_MAX_RECONNECT=${MESHSTREAM_MQTT_MAX_RECONNECT:-5m}
+      - MESHSTREAM_MQTT_USE_TLS=${MESHSTREAM_MQTT_USE_TLS:-false}
+      - MESHSTREAM_MQTT_TLS_PORT=${MESHSTREAM_MQTT_TLS_PORT:-8883}
      
      # Server configuration
      - MESHSTREAM_SERVER_HOST=${MESHSTREAM_SERVER_HOST:-0.0.0.0}
--- a/main.go
+++ b/main.go
@@ -19,11 +19,17 @@ import (
 // Config holds all the configuration parameters
 type Config struct {
 	// MQTT Configuration
-	MQTTBroker      string
-	MQTTUsername    string
-	MQTTPassword    string
-	MQTTTopicPrefix string
-	MQTTClientID    string
+	MQTTBroker         string
+	MQTTUsername       string
+	MQTTPassword       string
+	MQTTTopicPrefix    string
+	MQTTClientID       string
+	MQTTKeepAlive      int
+	MQTTConnectTimeout time.Duration
+	MQTTPingTimeout    time.Duration
+	MQTTMaxReconnect   time.Duration
+	MQTTUseTLS         bool
+	MQTTTLSPort        int

 	// Web server configuration
 	ServerHost string
@@ -69,6 +75,14 @@ func parseConfig() *Config {
 	flag.StringVar(&config.MQTTTopicPrefix, "mqtt-topic-prefix", getEnv("MQTT_TOPIC_PREFIX", "msh/US/CA/Motherlode"), "MQTT topic prefix")
 	flag.StringVar(&config.MQTTClientID, "mqtt-client-id", getEnv("MQTT_CLIENT_ID", "meshstream-client"), "MQTT client ID")

+	// MQTT connection tuning parameters
+	flag.IntVar(&config.MQTTKeepAlive, "mqtt-keepalive", intFromEnv("MQTT_KEEPALIVE", 60), "MQTT keep alive interval in seconds")
+	flag.DurationVar(&config.MQTTConnectTimeout, "mqtt-connect-timeout", durationFromEnv("MQTT_CONNECT_TIMEOUT", 30*time.Second), "MQTT connection timeout")
+	flag.DurationVar(&config.MQTTPingTimeout, "mqtt-ping-timeout", durationFromEnv("MQTT_PING_TIMEOUT", 10*time.Second), "MQTT ping timeout")
+	flag.DurationVar(&config.MQTTMaxReconnect, "mqtt-max-reconnect", durationFromEnv("MQTT_MAX_RECONNECT", 5*time.Minute), "MQTT maximum reconnect interval")
+	flag.BoolVar(&config.MQTTUseTLS, "mqtt-use-tls", boolFromEnv("MQTT_USE_TLS", false), "Use TLS for MQTT connection")
+	flag.IntVar(&config.MQTTTLSPort, "mqtt-tls-port", intFromEnv("MQTT_TLS_PORT", 8883), "MQTT TLS port")
+
 	// Web server configuration
 	flag.StringVar(&config.ServerHost, "server-host", getEnv("SERVER_HOST", "localhost"), "Web server host")
 	flag.StringVar(&config.ServerPort, "server-port", getEnv("SERVER_PORT", "8080"), "Web server port")
@@ -105,6 +119,20 @@ func mustParseDuration(durationStr string) time.Duration {
 	return duration
 }

+// Helper function to parse duration from environment with default
+func durationFromEnv(key string, defaultValue time.Duration) time.Duration {
+	envVal := getEnv(key, "")
+	if envVal == "" {
+		return defaultValue
+	}
+	duration, err := time.ParseDuration(envVal)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Invalid duration format for %s: %s\n", key, envVal)
+		return defaultValue
+	}
+	return duration
+}
+
 // Helper function to parse int from environment
 func intFromEnv(key string, defaultValue int) int {
 	envVal := getEnv(key, "")
@@ -156,7 +184,7 @@ func main() {

 		err := decoder.AddChannelKey(channelName, channelKey)
 		if err != nil {
-			logger.Errorw("Failed to initialize channel key", "channel", channelName, "error", err)
+			logger.Errorw("Failed to initialize channel key", "channel", channelName, "key", channelKey, "error", err)
 		} else {
 			logger.Infof("Initialized channel key for '%s'", channelName)
 		}
@@ -164,14 +192,22 @@ func main() {

 	// Configure and create the MQTT client
 	mqttConfig := mqtt.Config{
+		// Basic connection parameters
 		Broker:   config.MQTTBroker,
 		Username: config.MQTTUsername,
 		Password: config.MQTTPassword,
 		ClientID: config.MQTTClientID,
 		Topic:    config.MQTTTopicPrefix + "/#",
+
+		// Advanced connection parameters
+		KeepAlive:        config.MQTTKeepAlive,
+		ConnectTimeout:   config.MQTTConnectTimeout,
+		PingTimeout:      config.MQTTPingTimeout,
+		MaxReconnectTime: config.MQTTMaxReconnect,
+		UseTLS:           config.MQTTUseTLS,
+		TLSPort:          config.MQTTTLSPort,
 	}

-	logger.Infof("Connecting to MQTT broker: %s with topic prefix: %s", config.MQTTBroker, config.MQTTTopicPrefix)
 	mqttClient := mqtt.NewClient(mqttConfig, logger)

 	// Connect to the MQTT broker
--- a/mqtt/client.go
+++ b/mqtt/client.go
@@ -13,11 +13,20 @@ import (

 // Config holds configuration for the MQTT client
 type Config struct {
+	// Connection settings
 	Broker   string
 	Username string
 	Password string
 	ClientID string
 	Topic    string
+	
+	// Connection tuning parameters
+	KeepAlive        int           // Keep alive interval in seconds (default: 60)
+	ConnectTimeout   time.Duration // Connection timeout (default: 30s)
+	PingTimeout      time.Duration // Ping timeout (default: 10s)
+	MaxReconnectTime time.Duration // Maximum time between reconnect attempts (default: 5m)
+	UseTLS           bool          // Whether to use TLS/SSL (default: false)
+	TLSPort          int           // TLS port to use if UseTLS is true (default: 8883)
 }

 // Client manages the MQTT connection and message processing
@@ -41,26 +50,91 @@ func NewClient(config Config, logger logging.Logger) *Client {

 // Connect establishes a connection to the MQTT broker
 func (c *Client) Connect() error {
+	// Set default values for optional parameters
+	keepAlive := 60
+	if c.config.KeepAlive > 0 {
+		keepAlive = c.config.KeepAlive
+	}
+	
+	connectTimeout := 30 * time.Second
+	if c.config.ConnectTimeout > 0 {
+		connectTimeout = c.config.ConnectTimeout
+	}
+	
+	pingTimeout := 10 * time.Second
+	if c.config.PingTimeout > 0 {
+		pingTimeout = c.config.PingTimeout
+	}
+	
+	maxReconnectTime := 5 * time.Minute
+	if c.config.MaxReconnectTime > 0 {
+		maxReconnectTime = c.config.MaxReconnectTime
+	}
+	
+	// Determine protocol and port
+	protocol := "tcp"
+	port := 1883
+	if c.config.UseTLS {
+		protocol = "ssl"
+		port = 8883
+		if c.config.TLSPort > 0 {
+			port = c.config.TLSPort
+		}
+	}
+	
+	// Log detailed connection settings
+	c.logger.Infow("Connecting to MQTT broker with settings",
+		"broker", c.config.Broker,
+		"port", port,
+		"protocol", protocol,
+		"clientID", c.config.ClientID,
+		"username", c.config.Username,
+		"passwordLength", len(c.config.Password),
+		"topic", c.config.Topic,
+		"keepAlive", keepAlive,
+		"connectTimeout", connectTimeout,
+		"pingTimeout", pingTimeout,
+		"maxReconnectTime", maxReconnectTime,
+		"useTLS", c.config.UseTLS,
+	)
+
 	opts := mqtt.NewClientOptions()
-	opts.AddBroker(fmt.Sprintf("tcp://%s:1883", c.config.Broker))
+	opts.AddBroker(fmt.Sprintf("%s://%s:%d", protocol, c.config.Broker, port))
 	opts.SetClientID(c.config.ClientID)
 	opts.SetUsername(c.config.Username)
 	opts.SetPassword(c.config.Password)
 	opts.SetDefaultPublishHandler(c.messageHandler)
-	opts.SetPingTimeout(1 * time.Second)
+	opts.SetKeepAlive(time.Duration(keepAlive) * time.Second)
+	opts.SetConnectTimeout(connectTimeout)
+	opts.SetPingTimeout(pingTimeout)
+	opts.SetMaxReconnectInterval(maxReconnectTime)
+	opts.SetAutoReconnect(true)
+	opts.SetCleanSession(true)
 	opts.OnConnect = c.connectHandler
 	opts.OnConnectionLost = c.connectionLostHandler
+	opts.OnReconnecting = c.reconnectingHandler

 	// Create and start the client
 	c.client = mqtt.NewClient(opts)
 	if token := c.client.Connect(); token.Wait() && token.Error() != nil {
+		c.logger.Errorw("Failed to connect to MQTT broker", 
+			"error", token.Error(),
+			"broker", c.config.Broker,
+			"clientID", c.config.ClientID)
 		return fmt.Errorf("error connecting to MQTT broker: %v", token.Error())
 	}

 	// Subscribe to the configured topic
 	token := c.client.Subscribe(c.config.Topic, 0, nil)
 	token.Wait()
-	c.logger.Infof("Subscribed to topic: %s", c.config.Topic)
+	if token.Error() != nil {
+		c.logger.Errorw("Failed to subscribe to topic",
+			"error", token.Error(),
+			"topic", c.config.Topic)
+		return fmt.Errorf("error subscribing to topic %s: %v", c.config.Topic, token.Error())
+	}
+	
+	c.logger.Infof("Successfully subscribed to topic: %s", c.config.Topic)

 	return nil
 }
@@ -127,10 +201,24 @@ func (c *Client) messageHandler(client mqtt.Client, msg mqtt.Message) {

 // connectHandler is called when the client connects to the broker
 func (c *Client) connectHandler(client mqtt.Client) {
-	c.logger.Info("Connected to MQTT Broker")
+	c.logger.Infow("Connected to MQTT Broker", 
+		"broker", c.config.Broker,
+		"clientID", c.config.ClientID,
+		"topic", c.config.Topic)
 }

 // connectionLostHandler is called when the client loses connection
 func (c *Client) connectionLostHandler(client mqtt.Client, err error) {
-	c.logger.Errorw("Connection lost", "error", err)
+	c.logger.Errorw("Connection lost", 
+		"error", err,
+		"errorType", fmt.Sprintf("%T", err),
+		"broker", c.config.Broker,
+		"clientID", c.config.ClientID)
+}
+
+// reconnectingHandler is called when the client is attempting to reconnect
+func (c *Client) reconnectingHandler(client mqtt.Client, opts *mqtt.ClientOptions) {
+	c.logger.Infow("Attempting to reconnect to MQTT broker",
+		"broker", c.config.Broker,
+		"clientID", c.config.ClientID)
 }
--- a/scripts/push-to-ecr.sh
+++ b/scripts/push-to-ecr.sh
@@ -47,6 +47,13 @@ if ! docker buildx version &> /dev/null; then
 fi

 echo "Building Docker image for Meshstream using buildx (linux/amd64 platform)..."
+echo "Generating a unique MQTT client ID if not set..."
+# Generate a unique client ID if not already set
+if [ -z "${MESHSTREAM_MQTT_CLIENT_ID}" ]; then
+  export MESHSTREAM_MQTT_CLIENT_ID="meshstream-aws-$(date +%s)"
+  echo "Using generated MQTT client ID: ${MESHSTREAM_MQTT_CLIENT_ID}"
+fi
+
 docker buildx build \
  --platform linux/amd64 \
  --build-arg MESHSTREAM_API_BASE_URL="${MESHSTREAM_API_BASE_URL:-}" \
--- a/web/src/components/Nav.tsx
+++ b/web/src/components/Nav.tsx
@@ -1,7 +1,8 @@
-import React from "react";
+import React, { useState, useEffect } from "react";
 import { Link } from "@tanstack/react-router";
 import { ConnectionStatus } from "./ConnectionStatus";
 import { Separator } from "./Separator";
+import { Button } from "./ui/Button";
 import { SITE_TITLE } from "../lib/config";
 import {
  Info,
@@ -11,6 +12,8 @@ import {
  MessageSquare,
  Map,
  LucideIcon,
+  Menu,
+  X,
 } from "lucide-react";

 // Define navigation item structure
@@ -60,47 +63,124 @@ const navigationItems: NavItem[] = [
 ];

 export const Nav: React.FC<NavProps> = ({ connectionStatus }) => {
+  const [isMobileMenuOpen, setIsMobileMenuOpen] = useState(false);
+  const [isMobileView, setIsMobileView] = useState(false);
+
+  // Handle window resize to determine if we're in mobile view
+  useEffect(() => {
+    const handleResize = () => {
+      setIsMobileView(window.innerWidth < 768);
+      if (window.innerWidth >= 768) {
+        setIsMobileMenuOpen(false);
+      }
+    };
+
+    // Set initial state
+    handleResize();
+
+    // Add event listener
+    window.addEventListener("resize", handleResize);
+
+    // Clean up
+    return () => window.removeEventListener("resize", handleResize);
+  }, []);
+
+  // Navigation links component to avoid duplication
+  const NavLinks = () => (
+    <ul className="space-y-1">
+      {navigationItems.map((item) => (
+        <li key={item.to}>
+          <Link
+            to={item.to}
+            className="flex items-center px-4 py-2.5 transition-colors"
+            onClick={() => isMobileView && setIsMobileMenuOpen(false)}
+            inactiveProps={{
+              className:
+                "text-neutral-400 hover:text-neutral-200 font-thin",
+            }}
+            activeProps={{
+              className: "text-neutral-200 font-normal",
+            }}
+            activeOptions={{
+              exact: item.exact,
+            }}
+          >
+            <item.icon className="h-4 w-4 mr-3" />
+            {item.label}
+          </Link>
+        </li>
+      ))}
+    </ul>
+  );
+
  return (
-    <aside className="w-64 text-neutral-100 h-screen fixed left-0 top-0 flex flex-col">
-      {/* Logo and title section */}
-      <div className="p-4 flex items-center">
-        <div className="bg-pink-400 rounded-md mr-3 p-1.5 flex items-center justify-center">
-          <Layers className="h-5 w-5 text-neutral-800" />
+    <>
+      {/* Mobile header (always visible on mobile) */}
+      <div className="md:hidden fixed top-0 left-0 right-0 z-20  border-neutral-700 border-b shadow-inner bg-neutral-800/40">
+        <div className="flex items-center justify-between p-3">
+          <div className="flex items-center">
+            <div className="bg-pink-400 rounded-md mr-3 p-1.5 flex items-center justify-center">
+              <Layers className="h-5 w-5 text-neutral-800" />
+            </div>
+            <h1 className="text-xl font-thin tracking-wide text-neutral-100">{SITE_TITLE}</h1>
+          </div>
+          <Button 
+            variant="ghost" 
+            icon={isMobileMenuOpen ? X : Menu} 
+            onClick={() => setIsMobileMenuOpen(!isMobileMenuOpen)}
+            aria-label={isMobileMenuOpen ? "Close menu" : "Open menu"}
+          />
        </div>
-        <h1 className="text-xl font-thin tracking-wide">{SITE_TITLE}</h1>
      </div>

-      <Separator className="mt-1" />
+      {/* Mobile menu overlay */}
+      {isMobileMenuOpen && (
+        <div 
+          className="fixed inset-0 bg-black/50 z-30 md:hidden"
+          onClick={() => setIsMobileMenuOpen(false)}
+        />
+      )}

-      <nav className="flex-1">
-        <ul className="space-y-1">
-          {navigationItems.map((item) => (
-            <li key={item.to}>
-              <Link
-                to={item.to}
-                className="flex items-center px-4 py-2.5 transition-colors"
-                inactiveProps={{
-                  className:
-                    "text-neutral-400 hover:text-neutral-200 font-thin",
-                }}
-                activeProps={{
-                  className: "text-neutral-200 font-normal",
-                }}
-                activeOptions={{
-                  exact: item.exact,
-                }}
-              >
-                <item.icon className="h-4 w-4 mr-3" />
-                {item.label}
-              </Link>
-            </li>
-          ))}
-        </ul>
-      </nav>
+      {/* Mobile menu */}
+      <aside 
+        className={`
+          fixed top-0 right-0 h-screen bg-neutral-800 z-40 w-64 transform transition-transform duration-300 ease-in-out md:hidden
+          ${isMobileMenuOpen ? 'translate-x-0' : 'translate-x-full'}
+        `}
+      >
+        <div className="flex flex-col h-full pt-16">
+          <nav className="flex-1 overflow-y-auto">
+            <NavLinks />
+          </nav>
+          <div className="px-4 py-2 pb-6">
+            <ConnectionStatus status={connectionStatus} />
+          </div>
+        </div>
+      </aside>

-      <div className="px-4 py-2 pb-6">
-        <ConnectionStatus status={connectionStatus} />
-      </div>
-    </aside>
+      {/* Desktop sidebar (hidden on mobile) */}
+      <aside className="hidden md:flex w-64 text-neutral-100 h-screen fixed left-0 top-0 flex-col">
+        {/* Logo and title section */}
+        <div className="p-4 flex items-center">
+          <div className="bg-pink-400 rounded-md mr-3 p-1.5 flex items-center justify-center">
+            <Layers className="h-5 w-5 text-neutral-800" />
+          </div>
+          <h1 className="text-xl font-thin tracking-wide">{SITE_TITLE}</h1>
+        </div>
+
+        <Separator className="mt-1" />
+
+        <nav className="flex-1 overflow-y-auto">
+          <NavLinks />
+        </nav>
+
+        <div className="px-4 py-2 pb-6">
+          <ConnectionStatus status={connectionStatus} />
+        </div>
+      </aside>
+
+      {/* Content padding spacer for mobile view - this will be positioned outside the main content area */}
+      <div className="md:hidden h-16"></div>
+    </>
  );
 };
--- a/web/src/components/PageWrapper.tsx
+++ b/web/src/components/PageWrapper.tsx
@@ -9,7 +9,7 @@ interface PageWrapperProps {
 */
 export const PageWrapper: React.FC<PageWrapperProps> = ({ children }) => {
  return (
-    <div className="bg-neutral-50/5 flex-1 h-[calc(100vh-3rem)] overflow-y-auto rounded-tl-3xl rounded-bl-3xl p-4 shadow-md">
+    <div className="flex-1 h-[calc(100vh-6rem)] md:h-[calc(100vh-3rem)] overflow-y-auto md:rounded-tl-3xl md:rounded-bl-3xl p-4 shadow-md">
      {children}
    </div>
  );
--- a/web/src/components/ui/Button.tsx
+++ b/web/src/components/ui/Button.tsx
@@ -4,7 +4,7 @@ import { cn } from "../../lib/cn";

 export interface ButtonProps extends React.ButtonHTMLAttributes<HTMLButtonElement> {
  /** Button content */
-  children: React.ReactNode;
+  children?: React.ReactNode;
  /** Optional icon to display before the text */
  icon?: LucideIcon;
  /** Button variant */
--- a/web/src/routes/__root.tsx
+++ b/web/src/routes/__root.tsx
@@ -82,9 +82,9 @@ function RootLayout() {
  }, [dispatch]);

  return (
-    <div className="flex h-screen overflow-hidden bg-neutral-900">
+    <div className="flex h-screen overflow-hidden bg-neutral-800">
      <Nav connectionStatus={connectionStatus} />
-      <main className="ml-64 flex-1 py-6 overflow-hidden flex flex-col">
+      <main className="md:ml-64 flex-1 pt-16 md:pt-6 pb-2 md:py-6 overflow-hidden flex flex-col bg-neutral-50/5">
        <Outlet />
      </main>
    </div>
--- a/web/src/styles/index.css
+++ b/web/src/styles/index.css
@@ -5,7 +5,7 @@
 /* Set default background and text colors and font family */
@layer base {
  html, body {
-    @apply bg-neutral-800;
+    @apply bg-neutral-700;
    @apply text-neutral-200;
    @apply font-sans;
  }