mirror of
https://github.com/Cyclenerd/offline-map-tile-downloader.git
synced 2026-07-15 14:20:56 +02:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c91d609a17 | |||
| 82fdb3c803 | |||
| a9ab9c80cd | |||
| ded4de2ed0 | |||
| aa4a4bc188 | |||
| 128675600b |
@@ -12,7 +12,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 🛎️ Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
# https://github.com/marketplace/actions/setup-go-environment
|
||||
- name: 🔧 Setup go
|
||||
|
||||
@@ -11,7 +11,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: 🛎️ Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
# https://github.com/marketplace/actions/setup-go-environment
|
||||
- name: 🔧 Setup go
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: 🛎️ Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
# https://github.com/marketplace/actions/setup-go-environment
|
||||
- name: 🔧 Setup go
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
# Release, upload files
|
||||
# https://github.com/softprops/action-gh-release
|
||||
- name: ✨ Release
|
||||
uses: softprops/action-gh-release@v3.0.0
|
||||
uses: softprops/action-gh-release@v3.0.1
|
||||
with:
|
||||
files: |
|
||||
offline-map-tile-downloader-linux-x86_64
|
||||
|
||||
@@ -50,7 +50,7 @@ Don't worry, it's easier than it looks!
|
||||
1. Press the <kbd>Windows key</kbd> + <kbd>X</kbd>.
|
||||
1. Choose "Windows PowerShell".
|
||||
1. Type `cd` followed by the path to your your folder where you saved the application (e.g., `cd C:\Users\YourName\Documents\OfflineMaps`) and press Enter.
|
||||
1. Type `offline-map-tile-downloader.exe` and press Enter.
|
||||
1. Type `.\offline-map-tile-downloader.exe` and press Enter.
|
||||
1. Open your web browser and go to [`http://localhost:8080`](http://localhost:8080).
|
||||
|
||||
### <img src="https://developer.apple.com/favicon.ico" width=16 height=16/> macOS (Terminal)
|
||||
@@ -109,7 +109,7 @@ If you're a developer or want to modify the code, you can build the application
|
||||
|
||||
**Please use this tool responsibly!** When downloading map tiles:
|
||||
|
||||
* **Respect tile server limits:** Use reasonable rate limits to avoid overloading map tile servers. The default rate limit is 50 tiles per second, which is conservative for most servers.
|
||||
* **Respect tile server limits:** Use reasonable rate limits to avoid overloading map tile servers. The default rate limit is 10 tiles per second, which is conservative for most servers.
|
||||
* **Avoid excessive downloads:** Only download the areas and zoom levels you actually need. Downloading entire continents at high zoom levels can generate millions of tiles.
|
||||
* **Check terms of service:** Review the terms of service of the map tile provider you're using. Some providers have specific restrictions on bulk downloading.
|
||||
* **Risk of being banned:** Aggressive downloading patterns can result in your IP address being temporarily or permanently banned from tile servers.
|
||||
@@ -127,16 +127,33 @@ You can also use command-line options to configure the application:
|
||||
|
||||
* `-port`: The port number for the server (default: `8080`).
|
||||
* `-maps-directory`: The directory for cached map tiles (default: `maps`).
|
||||
* `-max-workers`: The number of concurrent download workers (default: `10`).
|
||||
* `-rate-limit`: The maximum number of tiles to download per second (default: `50`).
|
||||
* `-max-workers`: The number of concurrent download workers (default: `4`, max: `10`). Lower values are more respectful to tile servers.
|
||||
* `-rate-limit`: The maximum number of tiles to download per second (default: `10`, max: `50`). Keep this low to avoid being blocked.
|
||||
* `-max-retries`: The maximum number of retries for downloading a tile (default: `3`).
|
||||
* `-user-agent`: User-Agent header for HTTP requests (default: `MapTileDownloader/1.0 (Go)`).
|
||||
* `-user-agent`: User-Agent header for HTTP requests (default: `mesh/YYMMDD (OS)` where date changes daily).
|
||||
* `-help`: Show the help message.
|
||||
|
||||
**Being respectful to tile servers:**
|
||||
- Uses conservative default rate limits (10 tiles/sec, 4 workers)
|
||||
- Adds random delays (100-300ms) between requests to avoid hammering servers
|
||||
- Identifies itself honestly as a tile downloader
|
||||
- Respects retry limits with exponential backoff
|
||||
|
||||
**To reduce load further (recommended):**
|
||||
|
||||
```bash
|
||||
./offline-map-tile-downloader -max-workers 2 -rate-limit 5
|
||||
```
|
||||
|
||||
**Important:** Always respect the tile server's Terms of Service and usage policies. Many servers have usage limits. Consider:
|
||||
- Using lower rate limits for large downloads
|
||||
- Downloading during off-peak hours
|
||||
- Setting up your own tile server for heavy usage
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
./offline-map-tile-downloader -port 8081 -maps-directory my-tile-cache -max-workers 5 -rate-limit 25
|
||||
./offline-map-tile-downloader -port 8081 -maps-directory my-tile-cache -max-workers 2 -rate-limit 5
|
||||
```
|
||||
|
||||
## Meshtastic UI Integration
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -57,6 +58,26 @@ var (
|
||||
userAgent *string
|
||||
)
|
||||
|
||||
// generateUserAgent returns a basic user agent that identifies as a tile downloader
|
||||
func generateUserAgent() string {
|
||||
osName := runtime.GOOS
|
||||
var osInfo string
|
||||
|
||||
switch osName {
|
||||
case "darwin":
|
||||
osInfo = "Macintosh"
|
||||
case "linux":
|
||||
osInfo = "Linux"
|
||||
case "windows":
|
||||
osInfo = "Windows"
|
||||
default:
|
||||
osInfo = osName
|
||||
}
|
||||
|
||||
version := time.Now().Format("060102")
|
||||
return fmt.Sprintf("mesh/%s (%s)", version, osInfo)
|
||||
}
|
||||
|
||||
// Tile represents a single map tile with X, Y coordinates and zoom level Z.
|
||||
type Tile struct {
|
||||
X, Y, Z uint32
|
||||
@@ -99,10 +120,10 @@ func main() {
|
||||
// Command line flags
|
||||
port := flag.Int("port", 8080, "Port number for the server")
|
||||
cacheDir = flag.String("maps-directory", "maps", "Directory for storing map tiles. This is where the downloaded tiles will be saved.")
|
||||
maxWorkers = flag.Int("max-workers", 10, "Number of concurrent download workers")
|
||||
rateLimit = flag.Int("rate-limit", 50, "Maximum number of tiles to download per second")
|
||||
maxWorkers = flag.Int("max-workers", 4, "Number of concurrent download workers (max: 10)")
|
||||
rateLimit = flag.Int("rate-limit", 10, "Maximum number of tiles to download per second (max: 50)")
|
||||
maxRetries = flag.Int("max-retries", 3, "Maximum number of retries for downloading a tile")
|
||||
userAgent = flag.String("user-agent", "MapTileDownloader/1.0 (Go)", "User-Agent header for HTTP requests")
|
||||
userAgent = flag.String("user-agent", generateUserAgent(), "User-Agent header for HTTP requests")
|
||||
help := flag.Bool("help", false, "Show help message")
|
||||
|
||||
flag.Parse()
|
||||
@@ -112,6 +133,14 @@ func main() {
|
||||
return
|
||||
}
|
||||
|
||||
if *maxWorkers > 10 {
|
||||
log.Fatalf("max-workers cannot exceed 10 (got %d)", *maxWorkers)
|
||||
}
|
||||
|
||||
if *rateLimit > 50 {
|
||||
log.Fatalf("rate-limit cannot exceed 50 (got %d)", *rateLimit)
|
||||
}
|
||||
|
||||
// Create cache directory if it doesn't exist.
|
||||
if err := os.MkdirAll(*cacheDir, 0755); err != nil {
|
||||
log.Fatalf("Failed to create cache directory: %v", err)
|
||||
@@ -428,7 +457,13 @@ func downloadTile(ctx context.Context, msgChan chan<- WSMessage, tile Tile, mapS
|
||||
time.Sleep(time.Second * time.Duration(math.Pow(2, float64(attempt))))
|
||||
continue
|
||||
}
|
||||
|
||||
// Set basic headers
|
||||
req.Header.Set("User-Agent", *userAgent)
|
||||
req.Header.Set("Accept", "image/*")
|
||||
|
||||
// Add small random delay between requests (100-300ms)
|
||||
time.Sleep(time.Millisecond * time.Duration(100+rand.Intn(200)))
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user