mirror of https://github.com/aptly-dev/aptly
Use parallel gzip instead of gzip for compression
golangs compress/gzip isn't a parallel implementation, so it's quite a bit slower on most modern servers then pgzip. The below benchmark run shows that publishing a debian bullseye mirror snapshot (amd64, arm64, armhf, source) shows a gain of about 35% in publishing time (when skipping bz2 using MR #1081) ``` hyperfine -w 1 -m 3 -L aptly aptly-nobz2,aptly-nobz2-pgzip -p "{aptly} -config aptly.conf publish drop bullseye || true" "{aptly} -config aptly.conf publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye" Benchmark 1: aptly-nobz2 -config aptly.conf publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye Time (mean ± σ): 35.548 s ± 0.378 s [User: 39.465 s, System: 10.046 s] Range (min … max): 35.149 s … 35.902 s 3 runs Benchmark 2: aptly-nobz2-pgzip -config aptly.conf publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye Time (mean ± σ): 26.592 s ± 0.069 s [User: 42.207 s, System: 9.676 s] Range (min … max): 26.521 s … 26.660 s 3 runs Summary 'aptly-nobz2-pgzip -config aptly.conf publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye' ran 1.34 ± 0.01 times faster than 'aptly-nobz2 -config aptly.conf publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye' ``` Signed-off-by: Sjoerd Simons <sjoerd@collabora.com>pull/1081/head
parent
26254a0ad8
commit
2aca913e92
1
go.mod
1
go.mod
|
@ -17,6 +17,7 @@ require (
|
|||
github.com/jlaffaye/ftp v0.0.0-20180404123514-2403248fa8cc // indirect
|
||||
github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d
|
||||
github.com/klauspost/compress v1.13.6
|
||||
github.com/klauspost/pgzip v1.2.5
|
||||
github.com/mattn/go-colorable v0.1.2 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.2 // indirect
|
||||
|
|
2
go.sum
2
go.sum
|
@ -203,6 +203,8 @@ github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d h1:RnWZeH8N8KXfbwMTex/KKM
|
|||
github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d/go.mod h1:phT/jsRPBAEqjAibu1BurrabCBNTYiVI+zbmyCZJY6Q=
|
||||
github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
|
||||
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
|
||||
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
|
|
|
@ -1,10 +1,11 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
||||
"github.com/klauspost/pgzip"
|
||||
)
|
||||
|
||||
// CompressFile compresses file specified by source to .gz & .bz2
|
||||
|
@ -19,7 +20,7 @@ func CompressFile(source *os.File, onlyGzip bool) error {
|
|||
}
|
||||
defer gzFile.Close()
|
||||
|
||||
gzWriter := gzip.NewWriter(gzFile)
|
||||
gzWriter := pgzip.NewWriter(gzFile)
|
||||
defer gzWriter.Close()
|
||||
|
||||
source.Seek(0, 0)
|
||||
|
|
Loading…
Reference in New Issue