mirror of
https://github.com/Cyclenerd/offline-map-tile-downloader.git
synced 2026-08-10 10:42:49 +02:00
Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5396bed5b6 | |||
| 36565924e8 | |||
| 407bef4393 | |||
| 3bbf729872 | |||
| c91d609a17 | |||
| 82fdb3c803 | |||
| a9ab9c80cd | |||
| ded4de2ed0 | |||
| aa4a4bc188 | |||
| 128675600b |
@@ -12,11 +12,11 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: 🛎️ Checkout
|
- name: 🛎️ Checkout
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
# https://github.com/marketplace/actions/setup-go-environment
|
# https://github.com/marketplace/actions/setup-go-environment
|
||||||
- name: 🔧 Setup go
|
- name: 🔧 Setup go
|
||||||
uses: actions/setup-go@v6
|
uses: actions/setup-go@v7
|
||||||
|
|
||||||
- name: 🧝♀️ Check formatting
|
- name: 🧝♀️ Check formatting
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -11,11 +11,11 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
|
|
||||||
- name: 🛎️ Checkout
|
- name: 🛎️ Checkout
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
# https://github.com/marketplace/actions/setup-go-environment
|
# https://github.com/marketplace/actions/setup-go-environment
|
||||||
- name: 🔧 Setup go
|
- name: 🔧 Setup go
|
||||||
uses: actions/setup-go@v6
|
uses: actions/setup-go@v7
|
||||||
|
|
||||||
- name: 🍳 Build
|
- name: 🍳 Build
|
||||||
run: make
|
run: make
|
||||||
|
|||||||
@@ -13,11 +13,11 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
|
|
||||||
- name: 🛎️ Checkout
|
- name: 🛎️ Checkout
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
# https://github.com/marketplace/actions/setup-go-environment
|
# https://github.com/marketplace/actions/setup-go-environment
|
||||||
- name: 🔧 Setup go
|
- name: 🔧 Setup go
|
||||||
uses: actions/setup-go@v6
|
uses: actions/setup-go@v7
|
||||||
|
|
||||||
- name: 🍳 Build
|
- name: 🍳 Build
|
||||||
run: make
|
run: make
|
||||||
@@ -85,7 +85,7 @@ jobs:
|
|||||||
# Release, upload files
|
# Release, upload files
|
||||||
# https://github.com/softprops/action-gh-release
|
# https://github.com/softprops/action-gh-release
|
||||||
- name: ✨ Release
|
- name: ✨ Release
|
||||||
uses: softprops/action-gh-release@v3.0.0
|
uses: softprops/action-gh-release@v3.0.2
|
||||||
with:
|
with:
|
||||||
files: |
|
files: |
|
||||||
offline-map-tile-downloader-linux-x86_64
|
offline-map-tile-downloader-linux-x86_64
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ Don't worry, it's easier than it looks!
|
|||||||
1. Press the <kbd>Windows key</kbd> + <kbd>X</kbd>.
|
1. Press the <kbd>Windows key</kbd> + <kbd>X</kbd>.
|
||||||
1. Choose "Windows PowerShell".
|
1. Choose "Windows PowerShell".
|
||||||
1. Type `cd` followed by the path to your your folder where you saved the application (e.g., `cd C:\Users\YourName\Documents\OfflineMaps`) and press Enter.
|
1. Type `cd` followed by the path to your your folder where you saved the application (e.g., `cd C:\Users\YourName\Documents\OfflineMaps`) and press Enter.
|
||||||
1. Type `offline-map-tile-downloader.exe` and press Enter.
|
1. Type `.\offline-map-tile-downloader.exe` and press Enter.
|
||||||
1. Open your web browser and go to [`http://localhost:8080`](http://localhost:8080).
|
1. Open your web browser and go to [`http://localhost:8080`](http://localhost:8080).
|
||||||
|
|
||||||
### <img src="https://developer.apple.com/favicon.ico" width=16 height=16/> macOS (Terminal)
|
### <img src="https://developer.apple.com/favicon.ico" width=16 height=16/> macOS (Terminal)
|
||||||
@@ -109,7 +109,7 @@ If you're a developer or want to modify the code, you can build the application
|
|||||||
|
|
||||||
**Please use this tool responsibly!** When downloading map tiles:
|
**Please use this tool responsibly!** When downloading map tiles:
|
||||||
|
|
||||||
* **Respect tile server limits:** Use reasonable rate limits to avoid overloading map tile servers. The default rate limit is 50 tiles per second, which is conservative for most servers.
|
* **Respect tile server limits:** Use reasonable rate limits to avoid overloading map tile servers. The default rate limit is 10 tiles per second, which is conservative for most servers.
|
||||||
* **Avoid excessive downloads:** Only download the areas and zoom levels you actually need. Downloading entire continents at high zoom levels can generate millions of tiles.
|
* **Avoid excessive downloads:** Only download the areas and zoom levels you actually need. Downloading entire continents at high zoom levels can generate millions of tiles.
|
||||||
* **Check terms of service:** Review the terms of service of the map tile provider you're using. Some providers have specific restrictions on bulk downloading.
|
* **Check terms of service:** Review the terms of service of the map tile provider you're using. Some providers have specific restrictions on bulk downloading.
|
||||||
* **Risk of being banned:** Aggressive downloading patterns can result in your IP address being temporarily or permanently banned from tile servers.
|
* **Risk of being banned:** Aggressive downloading patterns can result in your IP address being temporarily or permanently banned from tile servers.
|
||||||
@@ -127,16 +127,33 @@ You can also use command-line options to configure the application:
|
|||||||
|
|
||||||
* `-port`: The port number for the server (default: `8080`).
|
* `-port`: The port number for the server (default: `8080`).
|
||||||
* `-maps-directory`: The directory for cached map tiles (default: `maps`).
|
* `-maps-directory`: The directory for cached map tiles (default: `maps`).
|
||||||
* `-max-workers`: The number of concurrent download workers (default: `10`).
|
* `-max-workers`: The number of concurrent download workers (default: `4`, max: `10`). Lower values are more respectful to tile servers.
|
||||||
* `-rate-limit`: The maximum number of tiles to download per second (default: `50`).
|
* `-rate-limit`: The maximum number of tiles to download per second (default: `10`, max: `50`). Keep this low to avoid being blocked.
|
||||||
* `-max-retries`: The maximum number of retries for downloading a tile (default: `3`).
|
* `-max-retries`: The maximum number of retries for downloading a tile (default: `3`).
|
||||||
* `-user-agent`: User-Agent header for HTTP requests (default: `MapTileDownloader/1.0 (Go)`).
|
* `-user-agent`: User-Agent header for HTTP requests (default: `mesh/YYMMDD (OS)` where date changes daily).
|
||||||
* `-help`: Show the help message.
|
* `-help`: Show the help message.
|
||||||
|
|
||||||
|
**Being respectful to tile servers:**
|
||||||
|
- Uses conservative default rate limits (10 tiles/sec, 4 workers)
|
||||||
|
- Adds random delays (100-300ms) between requests to avoid hammering servers
|
||||||
|
- Identifies itself honestly as a tile downloader
|
||||||
|
- Respects retry limits with exponential backoff
|
||||||
|
|
||||||
|
**To reduce load further (recommended):**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./offline-map-tile-downloader -max-workers 2 -rate-limit 5
|
||||||
|
```
|
||||||
|
|
||||||
|
**Important:** Always respect the tile server's Terms of Service and usage policies. Many servers have usage limits. Consider:
|
||||||
|
- Using lower rate limits for large downloads
|
||||||
|
- Downloading during off-peak hours
|
||||||
|
- Setting up your own tile server for heavy usage
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./offline-map-tile-downloader -port 8081 -maps-directory my-tile-cache -max-workers 5 -rate-limit 25
|
./offline-map-tile-downloader -port 8081 -maps-directory my-tile-cache -max-workers 2 -rate-limit 5
|
||||||
```
|
```
|
||||||
|
|
||||||
## Meshtastic UI Integration
|
## Meshtastic UI Integration
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -57,6 +58,26 @@ var (
|
|||||||
userAgent *string
|
userAgent *string
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// generateUserAgent returns a basic user agent that identifies as a tile downloader
|
||||||
|
func generateUserAgent() string {
|
||||||
|
osName := runtime.GOOS
|
||||||
|
var osInfo string
|
||||||
|
|
||||||
|
switch osName {
|
||||||
|
case "darwin":
|
||||||
|
osInfo = "Macintosh"
|
||||||
|
case "linux":
|
||||||
|
osInfo = "Linux"
|
||||||
|
case "windows":
|
||||||
|
osInfo = "Windows"
|
||||||
|
default:
|
||||||
|
osInfo = osName
|
||||||
|
}
|
||||||
|
|
||||||
|
version := time.Now().Format("060102")
|
||||||
|
return fmt.Sprintf("mesh/%s (%s)", version, osInfo)
|
||||||
|
}
|
||||||
|
|
||||||
// Tile represents a single map tile with X, Y coordinates and zoom level Z.
|
// Tile represents a single map tile with X, Y coordinates and zoom level Z.
|
||||||
type Tile struct {
|
type Tile struct {
|
||||||
X, Y, Z uint32
|
X, Y, Z uint32
|
||||||
@@ -99,10 +120,10 @@ func main() {
|
|||||||
// Command line flags
|
// Command line flags
|
||||||
port := flag.Int("port", 8080, "Port number for the server")
|
port := flag.Int("port", 8080, "Port number for the server")
|
||||||
cacheDir = flag.String("maps-directory", "maps", "Directory for storing map tiles. This is where the downloaded tiles will be saved.")
|
cacheDir = flag.String("maps-directory", "maps", "Directory for storing map tiles. This is where the downloaded tiles will be saved.")
|
||||||
maxWorkers = flag.Int("max-workers", 10, "Number of concurrent download workers")
|
maxWorkers = flag.Int("max-workers", 4, "Number of concurrent download workers (max: 10)")
|
||||||
rateLimit = flag.Int("rate-limit", 50, "Maximum number of tiles to download per second")
|
rateLimit = flag.Int("rate-limit", 10, "Maximum number of tiles to download per second (max: 50)")
|
||||||
maxRetries = flag.Int("max-retries", 3, "Maximum number of retries for downloading a tile")
|
maxRetries = flag.Int("max-retries", 3, "Maximum number of retries for downloading a tile")
|
||||||
userAgent = flag.String("user-agent", "MapTileDownloader/1.0 (Go)", "User-Agent header for HTTP requests")
|
userAgent = flag.String("user-agent", generateUserAgent(), "User-Agent header for HTTP requests")
|
||||||
help := flag.Bool("help", false, "Show help message")
|
help := flag.Bool("help", false, "Show help message")
|
||||||
|
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
@@ -112,6 +133,14 @@ func main() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if *maxWorkers > 10 {
|
||||||
|
log.Fatalf("max-workers cannot exceed 10 (got %d)", *maxWorkers)
|
||||||
|
}
|
||||||
|
|
||||||
|
if *rateLimit > 50 {
|
||||||
|
log.Fatalf("rate-limit cannot exceed 50 (got %d)", *rateLimit)
|
||||||
|
}
|
||||||
|
|
||||||
// Create cache directory if it doesn't exist.
|
// Create cache directory if it doesn't exist.
|
||||||
if err := os.MkdirAll(*cacheDir, 0755); err != nil {
|
if err := os.MkdirAll(*cacheDir, 0755); err != nil {
|
||||||
log.Fatalf("Failed to create cache directory: %v", err)
|
log.Fatalf("Failed to create cache directory: %v", err)
|
||||||
@@ -428,7 +457,13 @@ func downloadTile(ctx context.Context, msgChan chan<- WSMessage, tile Tile, mapS
|
|||||||
time.Sleep(time.Second * time.Duration(math.Pow(2, float64(attempt))))
|
time.Sleep(time.Second * time.Duration(math.Pow(2, float64(attempt))))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set basic headers
|
||||||
req.Header.Set("User-Agent", *userAgent)
|
req.Header.Set("User-Agent", *userAgent)
|
||||||
|
req.Header.Set("Accept", "image/*")
|
||||||
|
|
||||||
|
// Add small random delay between requests (100-300ms)
|
||||||
|
time.Sleep(time.Millisecond * time.Duration(100+rand.Intn(200)))
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
resp, err := http.DefaultClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
Reference in New Issue
Block a user