🌐 AI搜索 & 代理 主页
Google Git
Sign in
chromium / external / github.com / DataDog / zstd / refs/tags/v1.5.2 / . / zstd.go
blob: f792116f53cb09d47249316ad94c093c8452de91 [file] [log] [blame]
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]1package zstd
2
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]3/*
Evan Jones49e36672021-07-07 20:40:42[diff] [blame]4// support decoding of "legacy" zstd payloads from versions [0.4, 0.8], matching the
5// default configuration of the zstd command line tool:
6// https://github.com/facebook/zstd/blob/dev/programs/README.md
7#cgo CFLAGS: -DZSTD_LEGACY_SUPPORT=4
8
Vianney Tranf279bdb2016-02-23 19:23:52[diff] [blame]9#include "zstd.h"
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]10*/
11import "C"
12import (
Vianney Tran20de6362016-02-16 22:28:46[diff] [blame]13 "bytes"
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]14 "errors"
Vianney Tran20de6362016-02-16 22:28:46[diff] [blame]15 "io/ioutil"
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]16 "unsafe"
17)
18
Vianney Tran3b906112018-03-14 10:57:33[diff] [blame]19// Defines best and standard values for zstd cli
Jason Moiron3171a202016-07-06 19:41:10[diff] [blame]20const (
Vianney Tran3b906112018-03-14 10:57:33[diff] [blame]21 BestSpeed = 1
22 BestCompression = 20
23 DefaultCompression = 5
Jason Moiron3171a202016-07-06 19:41:10[diff] [blame]24)
25
26var (
Vianney Tran3b906112018-03-14 10:57:33[diff] [blame]27 // ErrEmptySlice is returned when there is nothing to compress
28 ErrEmptySlice = errors.New("Bytes slice is empty")
Jason Moiron3171a202016-07-06 19:41:10[diff] [blame]29)
30
Vianney Tran30c4b292022-04-06 19:56:29[diff] [blame]31const (
Vianney Tranf9731482022-04-14 14:56:25[diff] [blame]32 // decompressSizeBufferLimit is the limit we set on creating a decompression buffer for the Decompress API
33 // This is made to prevent DOS from maliciously-created payloads (aka zipbomb).
34 // For large payloads with a compression ratio > 10, you can do your own allocation and pass it to the method:
35 // dst := make([]byte, 1GB)
36 // decompressed, err := zstd.Decompress(dst, src)
37 decompressSizeBufferLimit = 1000 * 1000
38
Vianney Tran30c4b292022-04-06 19:56:29[diff] [blame]39 zstdFrameHeaderSizeMax = 18 // From zstd.h. Since it's experimental API, hardcoding it
40)
41
Jason Moiron3171a202016-07-06 19:41:10[diff] [blame]42// CompressBound returns the worst case size needed for a destination buffer,
43// which can be used to preallocate a destination buffer or select a previously
44// allocated buffer from a pool.
ghatdeva74c4162018-03-15 07:29:39[diff] [blame]45// See zstd.h to mirror implementation of ZSTD_COMPRESSBOUND
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]46func CompressBound(srcSize int) int {
ghatdeva74c4162018-03-15 07:29:39[diff] [blame]47 lowLimit := 128 << 10 // 128 kB
Elijah Andrews755efe72017-08-10 19:39:50[diff] [blame]48 var margin int
49 if srcSize < lowLimit {
ghatdeva74c4162018-03-15 07:29:39[diff] [blame]50 margin = (lowLimit - srcSize) >> 11
Elijah Andrews755efe72017-08-10 19:39:50[diff] [blame]51 }
52 return srcSize + (srcSize >> 8) + margin
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]53}
54
Jason Moiron3171a202016-07-06 19:41:10[diff] [blame]55// cCompressBound is a cgo call to check the go implementation above against the c code.
Vianney Tran20de6362016-02-16 22:28:46[diff] [blame]56func cCompressBound(srcSize int) int {
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]57 return int(C.ZSTD_compressBound(C.size_t(srcSize)))
58}
59
Vianney Tran30c4b292022-04-06 19:56:29[diff] [blame]60// decompressSizeHint tries to give a hint on how much of the output buffer size we should have
61// based on zstd frame descriptors. To prevent DOS from maliciously-created payloads, limit the size
62func decompressSizeHint(src []byte) int {
63 // 1 MB or 10x input size
64 upperBound := 10 * len(src)
Vianney Tranf9731482022-04-14 14:56:25[diff] [blame]65 if upperBound < decompressSizeBufferLimit {
66 upperBound = decompressSizeBufferLimit
Vianney Tran30c4b292022-04-06 19:56:29[diff] [blame]67 }
68
69 hint := upperBound
70 if len(src) >= zstdFrameHeaderSizeMax {
71 hint = int(C.ZSTD_getFrameContentSize(unsafe.Pointer(&src[0]), C.size_t(len(src))))
72 if hint < 0 { // On error, just use upperBound
73 hint = upperBound
74 }
75 }
76
77 // Take the minimum of both
78 if hint > upperBound {
79 return upperBound
80 }
81 return hint
82}
83
Jason Moiron3171a202016-07-06 19:41:10[diff] [blame]84// Compress src into dst. If you have a buffer to use, you can pass it to
85// prevent allocation. If it is too small, or if nil is passed, a new buffer
86// will be allocated and returned.
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]87func Compress(dst, src []byte) ([]byte, error) {
Jason Moiron3171a202016-07-06 19:41:10[diff] [blame]88 return CompressLevel(dst, src, DefaultCompression)
Vianney Tran0c568302016-02-16 22:02:31[diff] [blame]89}
90
Jason Moiron3171a202016-07-06 19:41:10[diff] [blame]91// CompressLevel is the same as Compress but you can pass a compression level
Vianney Tran0c568302016-02-16 22:02:31[diff] [blame]92func CompressLevel(dst, src []byte, level int) ([]byte, error) {
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]93 bound := CompressBound(len(src))
94 if cap(dst) >= bound {
95 dst = dst[0:bound] // Reuse dst buffer
96 } else {
97 dst = make([]byte, bound)
98 }
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]99
Evan Jones0ead11a2021-03-05 21:55:31[diff] [blame]100 // We need unsafe.Pointer(&src[0]) in the Cgo call to avoid "Go pointer to Go pointer" panics.
101 // This means we need to special case empty input. See:
102 // https://github.com/golang/go/issues/14210#issuecomment-346402945
103 var cWritten C.size_t
104 if len(src) == 0 {
105 cWritten = C.ZSTD_compress(
106 unsafe.Pointer(&dst[0]),
107 C.size_t(len(dst)),
108 unsafe.Pointer(nil),
109 C.size_t(0),
110 C.int(level))
111 } else {
112 cWritten = C.ZSTD_compress(
113 unsafe.Pointer(&dst[0]),
114 C.size_t(len(dst)),
115 unsafe.Pointer(&src[0]),
116 C.size_t(len(src)),
117 C.int(level))
Vianney Trana50a3a22018-10-01 16:51:03[diff] [blame]118 }
119
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]120 written := int(cWritten)
121 // Check if the return is an Error code
Vianney Tran0c568302016-02-16 22:02:31[diff] [blame]122 if err := getError(written); err != nil {
123 return nil, err
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]124 }
125 return dst[:written], nil
126}
127
Jason Moiron3171a202016-07-06 19:41:10[diff] [blame]128// Decompress src into dst. If you have a buffer to use, you can pass it to
129// prevent allocation. If it is too small, or if nil is passed, a new buffer
Josh Carter23ee9d32016-09-06 22:32:55[diff] [blame]130// will be allocated and returned.
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]131func Decompress(dst, src []byte) ([]byte, error) {
folbrich8bc925a2018-08-04 00:32:28[diff] [blame]132 if len(src) == 0 {
133 return []byte{}, ErrEmptySlice
134 }
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]135
Vianney Tran30c4b292022-04-06 19:56:29[diff] [blame]136 bound := decompressSizeHint(src)
137 if cap(dst) >= bound {
138 dst = dst[0:cap(dst)]
139 } else {
140 dst = make([]byte, bound)
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]141 }
Vianney Tran489b9112022-04-06 21:03:49[diff] [blame]142
143 written := int(C.ZSTD_decompress(
144 unsafe.Pointer(&dst[0]),
145 C.size_t(len(dst)),
146 unsafe.Pointer(&src[0]),
147 C.size_t(len(src))))
148 err := getError(written)
149 if err == nil {
150 return dst[:written], nil
151 }
152 if !IsDstSizeTooSmallError(err) {
153 return nil, err
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]154 }
Josh Carter23ee9d32016-09-06 22:32:55[diff] [blame]155
Vianney Tran20de6362016-02-16 22:28:46[diff] [blame]156 // We failed getting a dst buffer of correct size, use stream API
Jason Moiron3171a202016-07-06 19:41:10[diff] [blame]157 r := NewReader(bytes.NewReader(src))
158 defer r.Close()
159 return ioutil.ReadAll(r)
Vianney Tran97ccd392016-02-05 23:17:12[diff] [blame]160}