mirror of
https://github.com/kiddin9/op-packages.git
synced 2026-07-28 03:01:54 +08:00
129 lines
4.2 KiB
Diff
129 lines
4.2 KiB
Diff
From 0c6f9a36049423283052dfaf6eb9e211afe95c89 Mon Sep 17 00:00:00 2001
|
|
From: sbwml <admin@cooluc.com>
|
|
Date: Fri, 20 Mar 2026 10:11:24 +0800
|
|
Subject: [PATCH 3/3] perf(unpack): Use memory mapping to reduce memory usage
|
|
|
|
Replaced `os.ReadFile` with `mmap` to avoid loading the entire geoip.dat
|
|
or geosite.dat file into memory during unpacking. This fixes
|
|
out-of-memory errors on low-memory devices.
|
|
|
|
Signed-off-by: sbwml <admin@cooluc.com>
|
|
---
|
|
cmd/unpack/geoip.go | 18 ++++++++++++++----
|
|
cmd/unpack/geosite.go | 11 ++++++++++-
|
|
go.mod | 4 +++-
|
|
go.sum | 4 ++++
|
|
4 files changed, 31 insertions(+), 6 deletions(-)
|
|
|
|
--- a/cmd/unpack/geoip.go
|
|
+++ b/cmd/unpack/geoip.go
|
|
@@ -3,14 +3,16 @@ package unpack
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
- "github.com/spf13/cobra"
|
|
- "github.com/urlesistiana/v2dat/v2data"
|
|
- "go.uber.org/zap"
|
|
"io"
|
|
"net/netip"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
+
|
|
+ "github.com/edsrzf/mmap-go"
|
|
+ "github.com/spf13/cobra"
|
|
+ "github.com/urlesistiana/v2dat/v2data"
|
|
+ "go.uber.org/zap"
|
|
)
|
|
|
|
func newGeoIPCmd() *cobra.Command {
|
|
@@ -34,10 +36,18 @@ func newGeoIPCmd() *cobra.Command {
|
|
|
|
func unpackGeoIP(args *unpackArgs) error {
|
|
filePath, wantTags, ourDir := args.file, args.filters, args.outDir
|
|
- b, err := os.ReadFile(filePath)
|
|
+
|
|
+ file, err := os.Open(filePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
+ defer file.Close()
|
|
+
|
|
+ b, err := mmap.Map(file, mmap.RDONLY, 0)
|
|
+ if err != nil {
|
|
+ return fmt.Errorf("error mapping file: %w", err)
|
|
+ }
|
|
+ defer b.Unmap()
|
|
geoIPList, err := v2data.LoadGeoIPListFromDAT(b)
|
|
if err != nil {
|
|
return err
|
|
--- a/cmd/unpack/geosite.go
|
|
+++ b/cmd/unpack/geosite.go
|
|
@@ -3,6 +3,7 @@ package unpack
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
+ "github.com/edsrzf/mmap-go"
|
|
"github.com/spf13/cobra"
|
|
"github.com/urlesistiana/v2dat/v2data"
|
|
"go.uber.org/zap"
|
|
@@ -39,10 +40,18 @@ func newGeoSiteCmd() *cobra.Command {
|
|
|
|
func unpackGeoSite(args *unpackArgs) error {
|
|
filePath, suffixes, outDir := args.file, args.filters, args.outDir
|
|
- b, err := os.ReadFile(filePath)
|
|
+
|
|
+ file, err := os.Open(filePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
+ defer file.Close()
|
|
+
|
|
+ b, err := mmap.Map(file, mmap.RDONLY, 0)
|
|
+ if err != nil {
|
|
+ return fmt.Errorf("error mapping file: %w", err)
|
|
+ }
|
|
+ defer b.Unmap()
|
|
geoSiteList, err := v2data.LoadGeoSiteList(b)
|
|
if err != nil {
|
|
return err
|
|
--- a/go.mod
|
|
+++ b/go.mod
|
|
@@ -1,8 +1,9 @@
|
|
module github.com/urlesistiana/v2dat
|
|
|
|
-go 1.23
|
|
+go 1.25.0
|
|
|
|
require (
|
|
+ github.com/edsrzf/mmap-go v1.2.0
|
|
github.com/spf13/cobra v1.10.2
|
|
go.uber.org/zap v1.27.1
|
|
google.golang.org/protobuf v1.36.11
|
|
@@ -12,4 +13,5 @@ require (
|
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
|
github.com/spf13/pflag v1.0.10 // indirect
|
|
go.uber.org/multierr v1.11.0 // indirect
|
|
+ golang.org/x/sys v0.42.0 // indirect
|
|
)
|
|
--- a/go.sum
|
|
+++ b/go.sum
|
|
@@ -1,6 +1,8 @@
|
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
|
+github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84=
|
|
+github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
|
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
|
@@ -22,6 +24,8 @@ go.uber.org/multierr v1.11.0/go.mod h1:2
|
|
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
|
|
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
|
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
|
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
|
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
|
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
|
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|