encoder_options.go 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231
  1. package zstd
  2. import (
  3. "errors"
  4. "fmt"
  5. "runtime"
  6. "strings"
  7. )
  8. // EOption is an option for creating a encoder.
  9. type EOption func(*encoderOptions) error
  10. // options retains accumulated state of multiple options.
  11. type encoderOptions struct {
  12. concurrent int
  13. crc bool
  14. single *bool
  15. pad int
  16. blockSize int
  17. windowSize int
  18. level EncoderLevel
  19. fullZero bool
  20. noEntropy bool
  21. }
  22. func (o *encoderOptions) setDefault() {
  23. *o = encoderOptions{
  24. // use less ram: true for now, but may change.
  25. concurrent: runtime.GOMAXPROCS(0),
  26. crc: true,
  27. single: nil,
  28. blockSize: 1 << 16,
  29. windowSize: 1 << 22,
  30. level: SpeedDefault,
  31. }
  32. }
  33. // encoder returns an encoder with the selected options.
  34. func (o encoderOptions) encoder() encoder {
  35. switch o.level {
  36. case SpeedDefault:
  37. return &doubleFastEncoder{fastEncoder: fastEncoder{maxMatchOff: int32(o.windowSize)}}
  38. case SpeedFastest:
  39. return &fastEncoder{maxMatchOff: int32(o.windowSize)}
  40. }
  41. panic("unknown compression level")
  42. }
  43. // WithEncoderCRC will add CRC value to output.
  44. // Output will be 4 bytes larger.
  45. func WithEncoderCRC(b bool) EOption {
  46. return func(o *encoderOptions) error { o.crc = b; return nil }
  47. }
  48. // WithEncoderConcurrency will set the concurrency,
  49. // meaning the maximum number of decoders to run concurrently.
  50. // The value supplied must be at least 1.
  51. // By default this will be set to GOMAXPROCS.
  52. func WithEncoderConcurrency(n int) EOption {
  53. return func(o *encoderOptions) error {
  54. if n <= 0 {
  55. return fmt.Errorf("concurrency must be at least 1")
  56. }
  57. o.concurrent = n
  58. return nil
  59. }
  60. }
  61. // WithWindowSize will set the maximum allowed back-reference distance.
  62. // The value must be a power of two between WindowSizeMin and WindowSizeMax.
  63. // A larger value will enable better compression but allocate more memory and,
  64. // for above-default values, take considerably longer.
  65. // The default value is determined by the compression level.
  66. func WithWindowSize(n int) EOption {
  67. return func(o *encoderOptions) error {
  68. switch {
  69. case n < MinWindowSize:
  70. return fmt.Errorf("window size must be at least %d", MinWindowSize)
  71. case n > MaxWindowSize:
  72. return fmt.Errorf("window size must be at most %d", MaxWindowSize)
  73. case (n & (n - 1)) != 0:
  74. return errors.New("window size must be a power of 2")
  75. }
  76. o.windowSize = n
  77. if o.blockSize > o.windowSize {
  78. o.blockSize = o.windowSize
  79. }
  80. return nil
  81. }
  82. }
  83. // WithEncoderPadding will add padding to all output so the size will be a multiple of n.
  84. // This can be used to obfuscate the exact output size or make blocks of a certain size.
  85. // The contents will be a skippable frame, so it will be invisible by the decoder.
  86. // n must be > 0 and <= 1GB, 1<<30 bytes.
  87. // The padded area will be filled with data from crypto/rand.Reader.
  88. // If `EncodeAll` is used with data already in the destination, the total size will be multiple of this.
  89. func WithEncoderPadding(n int) EOption {
  90. return func(o *encoderOptions) error {
  91. if n <= 0 {
  92. return fmt.Errorf("padding must be at least 1")
  93. }
  94. // No need to waste our time.
  95. if n == 1 {
  96. o.pad = 0
  97. }
  98. if n > 1<<30 {
  99. return fmt.Errorf("padding must less than 1GB (1<<30 bytes) ")
  100. }
  101. o.pad = n
  102. return nil
  103. }
  104. }
  105. // EncoderLevel predefines encoder compression levels.
  106. // Only use the constants made available, since the actual mapping
  107. // of these values are very likely to change and your compression could change
  108. // unpredictably when upgrading the library.
  109. type EncoderLevel int
  110. const (
  111. speedNotSet EncoderLevel = iota
  112. // SpeedFastest will choose the fastest reasonable compression.
  113. // This is roughly equivalent to the fastest Zstandard mode.
  114. SpeedFastest
  115. // SpeedDefault is the default "pretty fast" compression option.
  116. // This is roughly equivalent to the default Zstandard mode (level 3).
  117. SpeedDefault
  118. // speedLast should be kept as the last actual compression option.
  119. // The is not for external usage, but is used to keep track of the valid options.
  120. speedLast
  121. // SpeedBetterCompression will (in the future) yield better compression than the default,
  122. // but at approximately 4x the CPU usage of the default.
  123. // For now this is not implemented.
  124. SpeedBetterCompression = SpeedDefault
  125. // SpeedBestCompression will choose the best available compression option.
  126. // For now this is not implemented.
  127. SpeedBestCompression = SpeedDefault
  128. )
  129. // EncoderLevelFromString will convert a string representation of an encoding level back
  130. // to a compression level. The compare is not case sensitive.
  131. // If the string wasn't recognized, (false, SpeedDefault) will be returned.
  132. func EncoderLevelFromString(s string) (bool, EncoderLevel) {
  133. for l := EncoderLevel(speedNotSet + 1); l < speedLast; l++ {
  134. if strings.EqualFold(s, l.String()) {
  135. return true, l
  136. }
  137. }
  138. return false, SpeedDefault
  139. }
  140. // EncoderLevelFromZstd will return an encoder level that closest matches the compression
  141. // ratio of a specific zstd compression level.
  142. // Many input values will provide the same compression level.
  143. func EncoderLevelFromZstd(level int) EncoderLevel {
  144. switch {
  145. case level < 3:
  146. return SpeedFastest
  147. case level >= 3:
  148. return SpeedDefault
  149. }
  150. return SpeedDefault
  151. }
  152. // String provides a string representation of the compression level.
  153. func (e EncoderLevel) String() string {
  154. switch e {
  155. case SpeedFastest:
  156. return "fastest"
  157. case SpeedDefault:
  158. return "default"
  159. default:
  160. return "invalid"
  161. }
  162. }
  163. // WithEncoderLevel specifies a predefined compression level.
  164. func WithEncoderLevel(l EncoderLevel) EOption {
  165. return func(o *encoderOptions) error {
  166. switch {
  167. case l <= speedNotSet || l >= speedLast:
  168. return fmt.Errorf("unknown encoder level")
  169. }
  170. o.level = l
  171. return nil
  172. }
  173. }
  174. // WithZeroFrames will encode 0 length input as full frames.
  175. // This can be needed for compatibility with zstandard usage,
  176. // but is not needed for this package.
  177. func WithZeroFrames(b bool) EOption {
  178. return func(o *encoderOptions) error {
  179. o.fullZero = b
  180. return nil
  181. }
  182. }
  183. // WithNoEntropyCompression will always skip entropy compression of literals.
  184. // This can be useful if content has matches, but unlikely to benefit from entropy
  185. // compression. Usually the slight speed improvement is not worth enabling this.
  186. func WithNoEntropyCompression(b bool) EOption {
  187. return func(o *encoderOptions) error {
  188. o.noEntropy = b
  189. return nil
  190. }
  191. }
  192. // WithSingleSegment will set the "single segment" flag when EncodeAll is used.
  193. // If this flag is set, data must be regenerated within a single continuous memory segment.
  194. // In this case, Window_Descriptor byte is skipped, but Frame_Content_Size is necessarily present.
  195. // As a consequence, the decoder must allocate a memory segment of size equal or larger than size of your content.
  196. // In order to preserve the decoder from unreasonable memory requirements,
  197. // a decoder is allowed to reject a compressed frame which requests a memory size beyond decoder's authorized range.
  198. // For broader compatibility, decoders are recommended to support memory sizes of at least 8 MB.
  199. // This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations.
  200. // If this is not specified, block encodes will automatically choose this based on the input size.
  201. // This setting has no effect on streamed encodes.
  202. func WithSingleSegment(b bool) EOption {
  203. return func(o *encoderOptions) error {
  204. o.single = &b
  205. return nil
  206. }
  207. }