123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231 |
- package zstd
- import (
- "errors"
- "fmt"
- "runtime"
- "strings"
- )
- // EOption is an option for creating a encoder.
- type EOption func(*encoderOptions) error
- // options retains accumulated state of multiple options.
- type encoderOptions struct {
- concurrent int
- crc bool
- single *bool
- pad int
- blockSize int
- windowSize int
- level EncoderLevel
- fullZero bool
- noEntropy bool
- }
- func (o *encoderOptions) setDefault() {
- *o = encoderOptions{
- // use less ram: true for now, but may change.
- concurrent: runtime.GOMAXPROCS(0),
- crc: true,
- single: nil,
- blockSize: 1 << 16,
- windowSize: 1 << 22,
- level: SpeedDefault,
- }
- }
- // encoder returns an encoder with the selected options.
- func (o encoderOptions) encoder() encoder {
- switch o.level {
- case SpeedDefault:
- return &doubleFastEncoder{fastEncoder: fastEncoder{maxMatchOff: int32(o.windowSize)}}
- case SpeedFastest:
- return &fastEncoder{maxMatchOff: int32(o.windowSize)}
- }
- panic("unknown compression level")
- }
- // WithEncoderCRC will add CRC value to output.
- // Output will be 4 bytes larger.
- func WithEncoderCRC(b bool) EOption {
- return func(o *encoderOptions) error { o.crc = b; return nil }
- }
- // WithEncoderConcurrency will set the concurrency,
- // meaning the maximum number of decoders to run concurrently.
- // The value supplied must be at least 1.
- // By default this will be set to GOMAXPROCS.
- func WithEncoderConcurrency(n int) EOption {
- return func(o *encoderOptions) error {
- if n <= 0 {
- return fmt.Errorf("concurrency must be at least 1")
- }
- o.concurrent = n
- return nil
- }
- }
- // WithWindowSize will set the maximum allowed back-reference distance.
- // The value must be a power of two between WindowSizeMin and WindowSizeMax.
- // A larger value will enable better compression but allocate more memory and,
- // for above-default values, take considerably longer.
- // The default value is determined by the compression level.
- func WithWindowSize(n int) EOption {
- return func(o *encoderOptions) error {
- switch {
- case n < MinWindowSize:
- return fmt.Errorf("window size must be at least %d", MinWindowSize)
- case n > MaxWindowSize:
- return fmt.Errorf("window size must be at most %d", MaxWindowSize)
- case (n & (n - 1)) != 0:
- return errors.New("window size must be a power of 2")
- }
- o.windowSize = n
- if o.blockSize > o.windowSize {
- o.blockSize = o.windowSize
- }
- return nil
- }
- }
- // WithEncoderPadding will add padding to all output so the size will be a multiple of n.
- // This can be used to obfuscate the exact output size or make blocks of a certain size.
- // The contents will be a skippable frame, so it will be invisible by the decoder.
- // n must be > 0 and <= 1GB, 1<<30 bytes.
- // The padded area will be filled with data from crypto/rand.Reader.
- // If `EncodeAll` is used with data already in the destination, the total size will be multiple of this.
- func WithEncoderPadding(n int) EOption {
- return func(o *encoderOptions) error {
- if n <= 0 {
- return fmt.Errorf("padding must be at least 1")
- }
- // No need to waste our time.
- if n == 1 {
- o.pad = 0
- }
- if n > 1<<30 {
- return fmt.Errorf("padding must less than 1GB (1<<30 bytes) ")
- }
- o.pad = n
- return nil
- }
- }
- // EncoderLevel predefines encoder compression levels.
- // Only use the constants made available, since the actual mapping
- // of these values are very likely to change and your compression could change
- // unpredictably when upgrading the library.
- type EncoderLevel int
- const (
- speedNotSet EncoderLevel = iota
- // SpeedFastest will choose the fastest reasonable compression.
- // This is roughly equivalent to the fastest Zstandard mode.
- SpeedFastest
- // SpeedDefault is the default "pretty fast" compression option.
- // This is roughly equivalent to the default Zstandard mode (level 3).
- SpeedDefault
- // speedLast should be kept as the last actual compression option.
- // The is not for external usage, but is used to keep track of the valid options.
- speedLast
- // SpeedBetterCompression will (in the future) yield better compression than the default,
- // but at approximately 4x the CPU usage of the default.
- // For now this is not implemented.
- SpeedBetterCompression = SpeedDefault
- // SpeedBestCompression will choose the best available compression option.
- // For now this is not implemented.
- SpeedBestCompression = SpeedDefault
- )
- // EncoderLevelFromString will convert a string representation of an encoding level back
- // to a compression level. The compare is not case sensitive.
- // If the string wasn't recognized, (false, SpeedDefault) will be returned.
- func EncoderLevelFromString(s string) (bool, EncoderLevel) {
- for l := EncoderLevel(speedNotSet + 1); l < speedLast; l++ {
- if strings.EqualFold(s, l.String()) {
- return true, l
- }
- }
- return false, SpeedDefault
- }
- // EncoderLevelFromZstd will return an encoder level that closest matches the compression
- // ratio of a specific zstd compression level.
- // Many input values will provide the same compression level.
- func EncoderLevelFromZstd(level int) EncoderLevel {
- switch {
- case level < 3:
- return SpeedFastest
- case level >= 3:
- return SpeedDefault
- }
- return SpeedDefault
- }
- // String provides a string representation of the compression level.
- func (e EncoderLevel) String() string {
- switch e {
- case SpeedFastest:
- return "fastest"
- case SpeedDefault:
- return "default"
- default:
- return "invalid"
- }
- }
- // WithEncoderLevel specifies a predefined compression level.
- func WithEncoderLevel(l EncoderLevel) EOption {
- return func(o *encoderOptions) error {
- switch {
- case l <= speedNotSet || l >= speedLast:
- return fmt.Errorf("unknown encoder level")
- }
- o.level = l
- return nil
- }
- }
- // WithZeroFrames will encode 0 length input as full frames.
- // This can be needed for compatibility with zstandard usage,
- // but is not needed for this package.
- func WithZeroFrames(b bool) EOption {
- return func(o *encoderOptions) error {
- o.fullZero = b
- return nil
- }
- }
- // WithNoEntropyCompression will always skip entropy compression of literals.
- // This can be useful if content has matches, but unlikely to benefit from entropy
- // compression. Usually the slight speed improvement is not worth enabling this.
- func WithNoEntropyCompression(b bool) EOption {
- return func(o *encoderOptions) error {
- o.noEntropy = b
- return nil
- }
- }
- // WithSingleSegment will set the "single segment" flag when EncodeAll is used.
- // If this flag is set, data must be regenerated within a single continuous memory segment.
- // In this case, Window_Descriptor byte is skipped, but Frame_Content_Size is necessarily present.
- // As a consequence, the decoder must allocate a memory segment of size equal or larger than size of your content.
- // In order to preserve the decoder from unreasonable memory requirements,
- // a decoder is allowed to reject a compressed frame which requests a memory size beyond decoder's authorized range.
- // For broader compatibility, decoders are recommended to support memory sizes of at least 8 MB.
- // This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations.
- // If this is not specified, block encodes will automatically choose this based on the input size.
- // This setting has no effect on streamed encodes.
- func WithSingleSegment(b bool) EOption {
- return func(o *encoderOptions) error {
- o.single = &b
- return nil
- }
- }
|