您最多选择25个主题 主题必须以字母或数字开头,可以包含连字符 (-),并且长度不得超过35个字符
 
 
 
 

904 行
24 KiB

  1. // Copyright 2009 The Go Authors. All rights reserved.
  2. // Copyright (c) 2015 Klaus Post
  3. // Use of this source code is governed by a BSD-style
  4. // license that can be found in the LICENSE file.
  5. package flate
  6. import (
  7. "encoding/binary"
  8. "fmt"
  9. "io"
  10. "math"
  11. )
  12. const (
  13. NoCompression = 0
  14. BestSpeed = 1
  15. BestCompression = 9
  16. DefaultCompression = -1
  17. // HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman
  18. // entropy encoding. This mode is useful in compressing data that has
  19. // already been compressed with an LZ style algorithm (e.g. Snappy or LZ4)
  20. // that lacks an entropy encoder. Compression gains are achieved when
  21. // certain bytes in the input stream occur more frequently than others.
  22. //
  23. // Note that HuffmanOnly produces a compressed output that is
  24. // RFC 1951 compliant. That is, any valid DEFLATE decompressor will
  25. // continue to be able to decompress this output.
  26. HuffmanOnly = -2
  27. ConstantCompression = HuffmanOnly // compatibility alias.
  28. logWindowSize = 15
  29. windowSize = 1 << logWindowSize
  30. windowMask = windowSize - 1
  31. logMaxOffsetSize = 15 // Standard DEFLATE
  32. minMatchLength = 4 // The smallest match that the compressor looks for
  33. maxMatchLength = 258 // The longest match for the compressor
  34. minOffsetSize = 1 // The shortest offset that makes any sense
  35. // The maximum number of tokens we will encode at the time.
  36. // Smaller sizes usually creates less optimal blocks.
  37. // Bigger can make context switching slow.
  38. // We use this for levels 7-9, so we make it big.
  39. maxFlateBlockTokens = 1 << 15
  40. maxStoreBlockSize = 65535
  41. hashBits = 17 // After 17 performance degrades
  42. hashSize = 1 << hashBits
  43. hashMask = (1 << hashBits) - 1
  44. hashShift = (hashBits + minMatchLength - 1) / minMatchLength
  45. maxHashOffset = 1 << 28
  46. skipNever = math.MaxInt32
  47. debugDeflate = false
  48. )
  49. type compressionLevel struct {
  50. good, lazy, nice, chain, fastSkipHashing, level int
  51. }
  52. // Compression levels have been rebalanced from zlib deflate defaults
  53. // to give a bigger spread in speed and compression.
  54. // See https://blog.klauspost.com/rebalancing-deflate-compression-levels/
  55. var levels = []compressionLevel{
  56. {}, // 0
  57. // Level 1-6 uses specialized algorithm - values not used
  58. {0, 0, 0, 0, 0, 1},
  59. {0, 0, 0, 0, 0, 2},
  60. {0, 0, 0, 0, 0, 3},
  61. {0, 0, 0, 0, 0, 4},
  62. {0, 0, 0, 0, 0, 5},
  63. {0, 0, 0, 0, 0, 6},
  64. // Levels 7-9 use increasingly more lazy matching
  65. // and increasingly stringent conditions for "good enough".
  66. {8, 12, 16, 24, skipNever, 7},
  67. {16, 30, 40, 64, skipNever, 8},
  68. {32, 258, 258, 1024, skipNever, 9},
  69. }
  70. // advancedState contains state for the advanced levels, with bigger hash tables, etc.
  71. type advancedState struct {
  72. // deflate state
  73. length int
  74. offset int
  75. maxInsertIndex int
  76. chainHead int
  77. hashOffset int
  78. ii uint16 // position of last match, intended to overflow to reset.
  79. // input window: unprocessed data is window[index:windowEnd]
  80. index int
  81. estBitsPerByte int
  82. hashMatch [maxMatchLength + minMatchLength]uint32
  83. // Input hash chains
  84. // hashHead[hashValue] contains the largest inputIndex with the specified hash value
  85. // If hashHead[hashValue] is within the current window, then
  86. // hashPrev[hashHead[hashValue] & windowMask] contains the previous index
  87. // with the same hash value.
  88. hashHead [hashSize]uint32
  89. hashPrev [windowSize]uint32
  90. }
  91. type compressor struct {
  92. compressionLevel
  93. h *huffmanEncoder
  94. w *huffmanBitWriter
  95. // compression algorithm
  96. fill func(*compressor, []byte) int // copy data to window
  97. step func(*compressor) // process window
  98. window []byte
  99. windowEnd int
  100. blockStart int // window index where current tokens start
  101. err error
  102. // queued output tokens
  103. tokens tokens
  104. fast fastEnc
  105. state *advancedState
  106. sync bool // requesting flush
  107. byteAvailable bool // if true, still need to process window[index-1].
  108. }
  109. func (d *compressor) fillDeflate(b []byte) int {
  110. s := d.state
  111. if s.index >= 2*windowSize-(minMatchLength+maxMatchLength) {
  112. // shift the window by windowSize
  113. copy(d.window[:], d.window[windowSize:2*windowSize])
  114. s.index -= windowSize
  115. d.windowEnd -= windowSize
  116. if d.blockStart >= windowSize {
  117. d.blockStart -= windowSize
  118. } else {
  119. d.blockStart = math.MaxInt32
  120. }
  121. s.hashOffset += windowSize
  122. if s.hashOffset > maxHashOffset {
  123. delta := s.hashOffset - 1
  124. s.hashOffset -= delta
  125. s.chainHead -= delta
  126. // Iterate over slices instead of arrays to avoid copying
  127. // the entire table onto the stack (Issue #18625).
  128. for i, v := range s.hashPrev[:] {
  129. if int(v) > delta {
  130. s.hashPrev[i] = uint32(int(v) - delta)
  131. } else {
  132. s.hashPrev[i] = 0
  133. }
  134. }
  135. for i, v := range s.hashHead[:] {
  136. if int(v) > delta {
  137. s.hashHead[i] = uint32(int(v) - delta)
  138. } else {
  139. s.hashHead[i] = 0
  140. }
  141. }
  142. }
  143. }
  144. n := copy(d.window[d.windowEnd:], b)
  145. d.windowEnd += n
  146. return n
  147. }
  148. func (d *compressor) writeBlock(tok *tokens, index int, eof bool) error {
  149. if index > 0 || eof {
  150. var window []byte
  151. if d.blockStart <= index {
  152. window = d.window[d.blockStart:index]
  153. }
  154. d.blockStart = index
  155. //d.w.writeBlock(tok, eof, window)
  156. d.w.writeBlockDynamic(tok, eof, window, d.sync)
  157. return d.w.err
  158. }
  159. return nil
  160. }
  161. // writeBlockSkip writes the current block and uses the number of tokens
  162. // to determine if the block should be stored on no matches, or
  163. // only huffman encoded.
  164. func (d *compressor) writeBlockSkip(tok *tokens, index int, eof bool) error {
  165. if index > 0 || eof {
  166. if d.blockStart <= index {
  167. window := d.window[d.blockStart:index]
  168. // If we removed less than a 64th of all literals
  169. // we huffman compress the block.
  170. if int(tok.n) > len(window)-int(tok.n>>6) {
  171. d.w.writeBlockHuff(eof, window, d.sync)
  172. } else {
  173. // Write a dynamic huffman block.
  174. d.w.writeBlockDynamic(tok, eof, window, d.sync)
  175. }
  176. } else {
  177. d.w.writeBlock(tok, eof, nil)
  178. }
  179. d.blockStart = index
  180. return d.w.err
  181. }
  182. return nil
  183. }
  184. // fillWindow will fill the current window with the supplied
  185. // dictionary and calculate all hashes.
  186. // This is much faster than doing a full encode.
  187. // Should only be used after a start/reset.
  188. func (d *compressor) fillWindow(b []byte) {
  189. // Do not fill window if we are in store-only or huffman mode.
  190. if d.level <= 0 {
  191. return
  192. }
  193. if d.fast != nil {
  194. // encode the last data, but discard the result
  195. if len(b) > maxMatchOffset {
  196. b = b[len(b)-maxMatchOffset:]
  197. }
  198. d.fast.Encode(&d.tokens, b)
  199. d.tokens.Reset()
  200. return
  201. }
  202. s := d.state
  203. // If we are given too much, cut it.
  204. if len(b) > windowSize {
  205. b = b[len(b)-windowSize:]
  206. }
  207. // Add all to window.
  208. n := copy(d.window[d.windowEnd:], b)
  209. // Calculate 256 hashes at the time (more L1 cache hits)
  210. loops := (n + 256 - minMatchLength) / 256
  211. for j := 0; j < loops; j++ {
  212. startindex := j * 256
  213. end := startindex + 256 + minMatchLength - 1
  214. if end > n {
  215. end = n
  216. }
  217. tocheck := d.window[startindex:end]
  218. dstSize := len(tocheck) - minMatchLength + 1
  219. if dstSize <= 0 {
  220. continue
  221. }
  222. dst := s.hashMatch[:dstSize]
  223. bulkHash4(tocheck, dst)
  224. var newH uint32
  225. for i, val := range dst {
  226. di := i + startindex
  227. newH = val & hashMask
  228. // Get previous value with the same hash.
  229. // Our chain should point to the previous value.
  230. s.hashPrev[di&windowMask] = s.hashHead[newH]
  231. // Set the head of the hash chain to us.
  232. s.hashHead[newH] = uint32(di + s.hashOffset)
  233. }
  234. }
  235. // Update window information.
  236. d.windowEnd += n
  237. s.index = n
  238. }
  239. // Try to find a match starting at index whose length is greater than prevSize.
  240. // We only look at chainCount possibilities before giving up.
  241. // pos = s.index, prevHead = s.chainHead-s.hashOffset, prevLength=minMatchLength-1, lookahead
  242. func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, offset int, ok bool) {
  243. minMatchLook := maxMatchLength
  244. if lookahead < minMatchLook {
  245. minMatchLook = lookahead
  246. }
  247. win := d.window[0 : pos+minMatchLook]
  248. // We quit when we get a match that's at least nice long
  249. nice := len(win) - pos
  250. if d.nice < nice {
  251. nice = d.nice
  252. }
  253. // If we've got a match that's good enough, only look in 1/4 the chain.
  254. tries := d.chain
  255. length = minMatchLength - 1
  256. wEnd := win[pos+length]
  257. wPos := win[pos:]
  258. minIndex := pos - windowSize
  259. if minIndex < 0 {
  260. minIndex = 0
  261. }
  262. offset = 0
  263. cGain := 0
  264. if d.chain < 100 {
  265. for i := prevHead; tries > 0; tries-- {
  266. if wEnd == win[i+length] {
  267. n := matchLen(win[i:i+minMatchLook], wPos)
  268. if n > length {
  269. length = n
  270. offset = pos - i
  271. ok = true
  272. if n >= nice {
  273. // The match is good enough that we don't try to find a better one.
  274. break
  275. }
  276. wEnd = win[pos+n]
  277. }
  278. }
  279. if i <= minIndex {
  280. // hashPrev[i & windowMask] has already been overwritten, so stop now.
  281. break
  282. }
  283. i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset
  284. if i < minIndex {
  285. break
  286. }
  287. }
  288. return
  289. }
  290. // Some like it higher (CSV), some like it lower (JSON)
  291. const baseCost = 6
  292. // Base is 4 bytes at with an additional cost.
  293. // Matches must be better than this.
  294. for i := prevHead; tries > 0; tries-- {
  295. if wEnd == win[i+length] {
  296. n := matchLen(win[i:i+minMatchLook], wPos)
  297. if n > length {
  298. // Calculate gain. Estimate
  299. newGain := d.h.bitLengthRaw(wPos[:n]) - int(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int(lengthExtraBits[lengthCodes[(n-3)&255]])
  300. //fmt.Println(n, "gain:", newGain, "prev:", cGain, "raw:", d.h.bitLengthRaw(wPos[:n]))
  301. if newGain > cGain {
  302. length = n
  303. offset = pos - i
  304. cGain = newGain
  305. ok = true
  306. if n >= nice {
  307. // The match is good enough that we don't try to find a better one.
  308. break
  309. }
  310. wEnd = win[pos+n]
  311. }
  312. }
  313. }
  314. if i <= minIndex {
  315. // hashPrev[i & windowMask] has already been overwritten, so stop now.
  316. break
  317. }
  318. i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset
  319. if i < minIndex {
  320. break
  321. }
  322. }
  323. return
  324. }
  325. func (d *compressor) writeStoredBlock(buf []byte) error {
  326. if d.w.writeStoredHeader(len(buf), false); d.w.err != nil {
  327. return d.w.err
  328. }
  329. d.w.writeBytes(buf)
  330. return d.w.err
  331. }
  332. // hash4 returns a hash representation of the first 4 bytes
  333. // of the supplied slice.
  334. // The caller must ensure that len(b) >= 4.
  335. func hash4(b []byte) uint32 {
  336. return hash4u(binary.LittleEndian.Uint32(b), hashBits)
  337. }
  338. // bulkHash4 will compute hashes using the same
  339. // algorithm as hash4
  340. func bulkHash4(b []byte, dst []uint32) {
  341. if len(b) < 4 {
  342. return
  343. }
  344. hb := binary.LittleEndian.Uint32(b)
  345. dst[0] = hash4u(hb, hashBits)
  346. end := len(b) - 4 + 1
  347. for i := 1; i < end; i++ {
  348. hb = (hb >> 8) | uint32(b[i+3])<<24
  349. dst[i] = hash4u(hb, hashBits)
  350. }
  351. }
  352. func (d *compressor) initDeflate() {
  353. d.window = make([]byte, 2*windowSize)
  354. d.byteAvailable = false
  355. d.err = nil
  356. if d.state == nil {
  357. return
  358. }
  359. s := d.state
  360. s.index = 0
  361. s.hashOffset = 1
  362. s.length = minMatchLength - 1
  363. s.offset = 0
  364. s.chainHead = -1
  365. }
  366. // deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever,
  367. // meaning it always has lazy matching on.
  368. func (d *compressor) deflateLazy() {
  369. s := d.state
  370. // Sanity enables additional runtime tests.
  371. // It's intended to be used during development
  372. // to supplement the currently ad-hoc unit tests.
  373. const sanity = debugDeflate
  374. if d.windowEnd-s.index < minMatchLength+maxMatchLength && !d.sync {
  375. return
  376. }
  377. if d.windowEnd != s.index && d.chain > 100 {
  378. // Get literal huffman coder.
  379. if d.h == nil {
  380. d.h = newHuffmanEncoder(maxFlateBlockTokens)
  381. }
  382. var tmp [256]uint16
  383. for _, v := range d.window[s.index:d.windowEnd] {
  384. tmp[v]++
  385. }
  386. d.h.generate(tmp[:], 15)
  387. }
  388. s.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
  389. for {
  390. if sanity && s.index > d.windowEnd {
  391. panic("index > windowEnd")
  392. }
  393. lookahead := d.windowEnd - s.index
  394. if lookahead < minMatchLength+maxMatchLength {
  395. if !d.sync {
  396. return
  397. }
  398. if sanity && s.index > d.windowEnd {
  399. panic("index > windowEnd")
  400. }
  401. if lookahead == 0 {
  402. // Flush current output block if any.
  403. if d.byteAvailable {
  404. // There is still one pending token that needs to be flushed
  405. d.tokens.AddLiteral(d.window[s.index-1])
  406. d.byteAvailable = false
  407. }
  408. if d.tokens.n > 0 {
  409. if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
  410. return
  411. }
  412. d.tokens.Reset()
  413. }
  414. return
  415. }
  416. }
  417. if s.index < s.maxInsertIndex {
  418. // Update the hash
  419. hash := hash4(d.window[s.index:])
  420. ch := s.hashHead[hash]
  421. s.chainHead = int(ch)
  422. s.hashPrev[s.index&windowMask] = ch
  423. s.hashHead[hash] = uint32(s.index + s.hashOffset)
  424. }
  425. prevLength := s.length
  426. prevOffset := s.offset
  427. s.length = minMatchLength - 1
  428. s.offset = 0
  429. minIndex := s.index - windowSize
  430. if minIndex < 0 {
  431. minIndex = 0
  432. }
  433. if s.chainHead-s.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy {
  434. if newLength, newOffset, ok := d.findMatch(s.index, s.chainHead-s.hashOffset, lookahead); ok {
  435. s.length = newLength
  436. s.offset = newOffset
  437. }
  438. }
  439. if prevLength >= minMatchLength && s.length <= prevLength {
  440. // Check for better match at end...
  441. //
  442. // checkOff must be >=2 since we otherwise risk checking s.index
  443. // Offset of 2 seems to yield best results.
  444. const checkOff = 2
  445. prevIndex := s.index - 1
  446. if prevIndex+prevLength+checkOff < s.maxInsertIndex {
  447. end := lookahead
  448. if lookahead > maxMatchLength {
  449. end = maxMatchLength
  450. }
  451. end += prevIndex
  452. idx := prevIndex + prevLength - (4 - checkOff)
  453. h := hash4(d.window[idx:])
  454. ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength + (4 - checkOff)
  455. if ch2 > minIndex {
  456. length := matchLen(d.window[prevIndex:end], d.window[ch2:])
  457. // It seems like a pure length metric is best.
  458. if length > prevLength {
  459. prevLength = length
  460. prevOffset = prevIndex - ch2
  461. }
  462. }
  463. }
  464. // There was a match at the previous step, and the current match is
  465. // not better. Output the previous match.
  466. d.tokens.AddMatch(uint32(prevLength-3), uint32(prevOffset-minOffsetSize))
  467. // Insert in the hash table all strings up to the end of the match.
  468. // index and index-1 are already inserted. If there is not enough
  469. // lookahead, the last two strings are not inserted into the hash
  470. // table.
  471. newIndex := s.index + prevLength - 1
  472. // Calculate missing hashes
  473. end := newIndex
  474. if end > s.maxInsertIndex {
  475. end = s.maxInsertIndex
  476. }
  477. end += minMatchLength - 1
  478. startindex := s.index + 1
  479. if startindex > s.maxInsertIndex {
  480. startindex = s.maxInsertIndex
  481. }
  482. tocheck := d.window[startindex:end]
  483. dstSize := len(tocheck) - minMatchLength + 1
  484. if dstSize > 0 {
  485. dst := s.hashMatch[:dstSize]
  486. bulkHash4(tocheck, dst)
  487. var newH uint32
  488. for i, val := range dst {
  489. di := i + startindex
  490. newH = val & hashMask
  491. // Get previous value with the same hash.
  492. // Our chain should point to the previous value.
  493. s.hashPrev[di&windowMask] = s.hashHead[newH]
  494. // Set the head of the hash chain to us.
  495. s.hashHead[newH] = uint32(di + s.hashOffset)
  496. }
  497. }
  498. s.index = newIndex
  499. d.byteAvailable = false
  500. s.length = minMatchLength - 1
  501. if d.tokens.n == maxFlateBlockTokens {
  502. // The block includes the current character
  503. if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
  504. return
  505. }
  506. d.tokens.Reset()
  507. }
  508. s.ii = 0
  509. } else {
  510. // Reset, if we got a match this run.
  511. if s.length >= minMatchLength {
  512. s.ii = 0
  513. }
  514. // We have a byte waiting. Emit it.
  515. if d.byteAvailable {
  516. s.ii++
  517. d.tokens.AddLiteral(d.window[s.index-1])
  518. if d.tokens.n == maxFlateBlockTokens {
  519. if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
  520. return
  521. }
  522. d.tokens.Reset()
  523. }
  524. s.index++
  525. // If we have a long run of no matches, skip additional bytes
  526. // Resets when s.ii overflows after 64KB.
  527. if n := int(s.ii) - d.chain; n > 0 {
  528. n = 1 + int(n>>6)
  529. for j := 0; j < n; j++ {
  530. if s.index >= d.windowEnd-1 {
  531. break
  532. }
  533. d.tokens.AddLiteral(d.window[s.index-1])
  534. if d.tokens.n == maxFlateBlockTokens {
  535. if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
  536. return
  537. }
  538. d.tokens.Reset()
  539. }
  540. // Index...
  541. if s.index < s.maxInsertIndex {
  542. h := hash4(d.window[s.index:])
  543. ch := s.hashHead[h]
  544. s.chainHead = int(ch)
  545. s.hashPrev[s.index&windowMask] = ch
  546. s.hashHead[h] = uint32(s.index + s.hashOffset)
  547. }
  548. s.index++
  549. }
  550. // Flush last byte
  551. d.tokens.AddLiteral(d.window[s.index-1])
  552. d.byteAvailable = false
  553. // s.length = minMatchLength - 1 // not needed, since s.ii is reset above, so it should never be > minMatchLength
  554. if d.tokens.n == maxFlateBlockTokens {
  555. if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
  556. return
  557. }
  558. d.tokens.Reset()
  559. }
  560. }
  561. } else {
  562. s.index++
  563. d.byteAvailable = true
  564. }
  565. }
  566. }
  567. }
  568. func (d *compressor) store() {
  569. if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) {
  570. d.err = d.writeStoredBlock(d.window[:d.windowEnd])
  571. d.windowEnd = 0
  572. }
  573. }
  574. // fillWindow will fill the buffer with data for huffman-only compression.
  575. // The number of bytes copied is returned.
  576. func (d *compressor) fillBlock(b []byte) int {
  577. n := copy(d.window[d.windowEnd:], b)
  578. d.windowEnd += n
  579. return n
  580. }
  581. // storeHuff will compress and store the currently added data,
  582. // if enough has been accumulated or we at the end of the stream.
  583. // Any error that occurred will be in d.err
  584. func (d *compressor) storeHuff() {
  585. if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 {
  586. return
  587. }
  588. d.w.writeBlockHuff(false, d.window[:d.windowEnd], d.sync)
  589. d.err = d.w.err
  590. d.windowEnd = 0
  591. }
  592. // storeFast will compress and store the currently added data,
  593. // if enough has been accumulated or we at the end of the stream.
  594. // Any error that occurred will be in d.err
  595. func (d *compressor) storeFast() {
  596. // We only compress if we have maxStoreBlockSize.
  597. if d.windowEnd < len(d.window) {
  598. if !d.sync {
  599. return
  600. }
  601. // Handle extremely small sizes.
  602. if d.windowEnd < 128 {
  603. if d.windowEnd == 0 {
  604. return
  605. }
  606. if d.windowEnd <= 32 {
  607. d.err = d.writeStoredBlock(d.window[:d.windowEnd])
  608. } else {
  609. d.w.writeBlockHuff(false, d.window[:d.windowEnd], true)
  610. d.err = d.w.err
  611. }
  612. d.tokens.Reset()
  613. d.windowEnd = 0
  614. d.fast.Reset()
  615. return
  616. }
  617. }
  618. d.fast.Encode(&d.tokens, d.window[:d.windowEnd])
  619. // If we made zero matches, store the block as is.
  620. if d.tokens.n == 0 {
  621. d.err = d.writeStoredBlock(d.window[:d.windowEnd])
  622. // If we removed less than 1/16th, huffman compress the block.
  623. } else if int(d.tokens.n) > d.windowEnd-(d.windowEnd>>4) {
  624. d.w.writeBlockHuff(false, d.window[:d.windowEnd], d.sync)
  625. d.err = d.w.err
  626. } else {
  627. d.w.writeBlockDynamic(&d.tokens, false, d.window[:d.windowEnd], d.sync)
  628. d.err = d.w.err
  629. }
  630. d.tokens.Reset()
  631. d.windowEnd = 0
  632. }
  633. // write will add input byte to the stream.
  634. // Unless an error occurs all bytes will be consumed.
  635. func (d *compressor) write(b []byte) (n int, err error) {
  636. if d.err != nil {
  637. return 0, d.err
  638. }
  639. n = len(b)
  640. for len(b) > 0 {
  641. if d.windowEnd == len(d.window) || d.sync {
  642. d.step(d)
  643. }
  644. b = b[d.fill(d, b):]
  645. if d.err != nil {
  646. return 0, d.err
  647. }
  648. }
  649. return n, d.err
  650. }
  651. func (d *compressor) syncFlush() error {
  652. d.sync = true
  653. if d.err != nil {
  654. return d.err
  655. }
  656. d.step(d)
  657. if d.err == nil {
  658. d.w.writeStoredHeader(0, false)
  659. d.w.flush()
  660. d.err = d.w.err
  661. }
  662. d.sync = false
  663. return d.err
  664. }
  665. func (d *compressor) init(w io.Writer, level int) (err error) {
  666. d.w = newHuffmanBitWriter(w)
  667. switch {
  668. case level == NoCompression:
  669. d.window = make([]byte, maxStoreBlockSize)
  670. d.fill = (*compressor).fillBlock
  671. d.step = (*compressor).store
  672. case level == ConstantCompression:
  673. d.w.logNewTablePenalty = 10
  674. d.window = make([]byte, 32<<10)
  675. d.fill = (*compressor).fillBlock
  676. d.step = (*compressor).storeHuff
  677. case level == DefaultCompression:
  678. level = 5
  679. fallthrough
  680. case level >= 1 && level <= 6:
  681. d.w.logNewTablePenalty = 7
  682. d.fast = newFastEnc(level)
  683. d.window = make([]byte, maxStoreBlockSize)
  684. d.fill = (*compressor).fillBlock
  685. d.step = (*compressor).storeFast
  686. case 7 <= level && level <= 9:
  687. d.w.logNewTablePenalty = 8
  688. d.state = &advancedState{}
  689. d.compressionLevel = levels[level]
  690. d.initDeflate()
  691. d.fill = (*compressor).fillDeflate
  692. d.step = (*compressor).deflateLazy
  693. default:
  694. return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level)
  695. }
  696. d.level = level
  697. return nil
  698. }
  699. // reset the state of the compressor.
  700. func (d *compressor) reset(w io.Writer) {
  701. d.w.reset(w)
  702. d.sync = false
  703. d.err = nil
  704. // We only need to reset a few things for Snappy.
  705. if d.fast != nil {
  706. d.fast.Reset()
  707. d.windowEnd = 0
  708. d.tokens.Reset()
  709. return
  710. }
  711. switch d.compressionLevel.chain {
  712. case 0:
  713. // level was NoCompression or ConstantCompresssion.
  714. d.windowEnd = 0
  715. default:
  716. s := d.state
  717. s.chainHead = -1
  718. for i := range s.hashHead {
  719. s.hashHead[i] = 0
  720. }
  721. for i := range s.hashPrev {
  722. s.hashPrev[i] = 0
  723. }
  724. s.hashOffset = 1
  725. s.index, d.windowEnd = 0, 0
  726. d.blockStart, d.byteAvailable = 0, false
  727. d.tokens.Reset()
  728. s.length = minMatchLength - 1
  729. s.offset = 0
  730. s.ii = 0
  731. s.maxInsertIndex = 0
  732. }
  733. }
  734. func (d *compressor) close() error {
  735. if d.err != nil {
  736. return d.err
  737. }
  738. d.sync = true
  739. d.step(d)
  740. if d.err != nil {
  741. return d.err
  742. }
  743. if d.w.writeStoredHeader(0, true); d.w.err != nil {
  744. return d.w.err
  745. }
  746. d.w.flush()
  747. d.w.reset(nil)
  748. return d.w.err
  749. }
  750. // NewWriter returns a new Writer compressing data at the given level.
  751. // Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression);
  752. // higher levels typically run slower but compress more.
  753. // Level 0 (NoCompression) does not attempt any compression; it only adds the
  754. // necessary DEFLATE framing.
  755. // Level -1 (DefaultCompression) uses the default compression level.
  756. // Level -2 (ConstantCompression) will use Huffman compression only, giving
  757. // a very fast compression for all types of input, but sacrificing considerable
  758. // compression efficiency.
  759. //
  760. // If level is in the range [-2, 9] then the error returned will be nil.
  761. // Otherwise the error returned will be non-nil.
  762. func NewWriter(w io.Writer, level int) (*Writer, error) {
  763. var dw Writer
  764. if err := dw.d.init(w, level); err != nil {
  765. return nil, err
  766. }
  767. return &dw, nil
  768. }
  769. // NewWriterDict is like NewWriter but initializes the new
  770. // Writer with a preset dictionary. The returned Writer behaves
  771. // as if the dictionary had been written to it without producing
  772. // any compressed output. The compressed data written to w
  773. // can only be decompressed by a Reader initialized with the
  774. // same dictionary.
  775. func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
  776. zw, err := NewWriter(w, level)
  777. if err != nil {
  778. return nil, err
  779. }
  780. zw.d.fillWindow(dict)
  781. zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method.
  782. return zw, err
  783. }
  784. // A Writer takes data written to it and writes the compressed
  785. // form of that data to an underlying writer (see NewWriter).
  786. type Writer struct {
  787. d compressor
  788. dict []byte
  789. }
  790. // Write writes data to w, which will eventually write the
  791. // compressed form of data to its underlying writer.
  792. func (w *Writer) Write(data []byte) (n int, err error) {
  793. return w.d.write(data)
  794. }
  795. // Flush flushes any pending data to the underlying writer.
  796. // It is useful mainly in compressed network protocols, to ensure that
  797. // a remote reader has enough data to reconstruct a packet.
  798. // Flush does not return until the data has been written.
  799. // Calling Flush when there is no pending data still causes the Writer
  800. // to emit a sync marker of at least 4 bytes.
  801. // If the underlying writer returns an error, Flush returns that error.
  802. //
  803. // In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
  804. func (w *Writer) Flush() error {
  805. // For more about flushing:
  806. // http://www.bolet.org/~pornin/deflate-flush.html
  807. return w.d.syncFlush()
  808. }
  809. // Close flushes and closes the writer.
  810. func (w *Writer) Close() error {
  811. return w.d.close()
  812. }
  813. // Reset discards the writer's state and makes it equivalent to
  814. // the result of NewWriter or NewWriterDict called with dst
  815. // and w's level and dictionary.
  816. func (w *Writer) Reset(dst io.Writer) {
  817. if len(w.dict) > 0 {
  818. // w was created with NewWriterDict
  819. w.d.reset(dst)
  820. if dst != nil {
  821. w.d.fillWindow(w.dict)
  822. }
  823. } else {
  824. // w was created with NewWriter
  825. w.d.reset(dst)
  826. }
  827. }
  828. // ResetDict discards the writer's state and makes it equivalent to
  829. // the result of NewWriter or NewWriterDict called with dst
  830. // and w's level, but sets a specific dictionary.
  831. func (w *Writer) ResetDict(dst io.Writer, dict []byte) {
  832. w.dict = dict
  833. w.d.reset(dst)
  834. w.d.fillWindow(w.dict)
  835. }