001 /*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied. See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019 package org.apache.commons.compress.compressors.bzip2;
020
021 import java.io.IOException;
022 import java.io.OutputStream;
023
024 import org.apache.commons.compress.compressors.CompressorOutputStream;
025
026 /**
027 * An output stream that compresses into the BZip2 format into another stream.
028 *
029 * <p>
030 * The compression requires large amounts of memory. Thus you should call the
031 * {@link #close() close()} method as soon as possible, to force
032 * <tt>BZip2CompressorOutputStream</tt> to release the allocated memory.
033 * </p>
034 *
035 * <p> You can shrink the amount of allocated memory and maybe raise
036 * the compression speed by choosing a lower blocksize, which in turn
037 * may cause a lower compression ratio. You can avoid unnecessary
038 * memory allocation by avoiding using a blocksize which is bigger
039 * than the size of the input. </p>
040 *
041 * <p> You can compute the memory usage for compressing by the
042 * following formula: </p>
043 *
044 * <pre>
045 * <code>400k + (9 * blocksize)</code>.
046 * </pre>
047 *
048 * <p> To get the memory required for decompression by {@link
049 * BZip2CompressorInputStream} use </p>
050 *
051 * <pre>
052 * <code>65k + (5 * blocksize)</code>.
053 * </pre>
054 *
055 * <table width="100%" border="1">
056 * <colgroup> <col width="33%" /> <col width="33%" /> <col width="33%" />
057 * </colgroup>
058 * <tr>
059 * <th colspan="3">Memory usage by blocksize</th>
060 * </tr>
061 * <tr>
062 * <th align="right">Blocksize</th> <th align="right">Compression<br>
063 * memory usage</th> <th align="right">Decompression<br>
064 * memory usage</th>
065 * </tr>
066 * <tr>
067 * <td align="right">100k</td>
068 * <td align="right">1300k</td>
069 * <td align="right">565k</td>
070 * </tr>
071 * <tr>
072 * <td align="right">200k</td>
073 * <td align="right">2200k</td>
074 * <td align="right">1065k</td>
075 * </tr>
076 * <tr>
077 * <td align="right">300k</td>
078 * <td align="right">3100k</td>
079 * <td align="right">1565k</td>
080 * </tr>
081 * <tr>
082 * <td align="right">400k</td>
083 * <td align="right">4000k</td>
084 * <td align="right">2065k</td>
085 * </tr>
086 * <tr>
087 * <td align="right">500k</td>
088 * <td align="right">4900k</td>
089 * <td align="right">2565k</td>
090 * </tr>
091 * <tr>
092 * <td align="right">600k</td>
093 * <td align="right">5800k</td>
094 * <td align="right">3065k</td>
095 * </tr>
096 * <tr>
097 * <td align="right">700k</td>
098 * <td align="right">6700k</td>
099 * <td align="right">3565k</td>
100 * </tr>
101 * <tr>
102 * <td align="right">800k</td>
103 * <td align="right">7600k</td>
104 * <td align="right">4065k</td>
105 * </tr>
106 * <tr>
107 * <td align="right">900k</td>
108 * <td align="right">8500k</td>
109 * <td align="right">4565k</td>
110 * </tr>
111 * </table>
112 *
113 * <p>
114 * For decompression <tt>BZip2CompressorInputStream</tt> allocates less memory if the
115 * bzipped input is smaller than one block.
116 * </p>
117 *
118 * <p>
119 * Instances of this class are not threadsafe.
120 * </p>
121 *
122 * <p>
123 * TODO: Update to BZip2 1.0.1
124 * </p>
125 * @NotThreadSafe
126 */
127 public class BZip2CompressorOutputStream extends CompressorOutputStream
128 implements BZip2Constants {
129
130 /**
131 * The minimum supported blocksize <tt> == 1</tt>.
132 */
133 public static final int MIN_BLOCKSIZE = 1;
134
135 /**
136 * The maximum supported blocksize <tt> == 9</tt>.
137 */
138 public static final int MAX_BLOCKSIZE = 9;
139
140 private static final int SETMASK = (1 << 21);
141 private static final int CLEARMASK = (~SETMASK);
142 private static final int GREATER_ICOST = 15;
143 private static final int LESSER_ICOST = 0;
144 private static final int SMALL_THRESH = 20;
145 private static final int DEPTH_THRESH = 10;
146 private static final int WORK_FACTOR = 30;
147
148 /*
149 * <p> If you are ever unlucky/improbable enough to get a stack
150 * overflow whilst sorting, increase the following constant and
151 * try again. In practice I have never seen the stack go above 27
152 * elems, so the following limit seems very generous. </p>
153 */
154 private static final int QSORT_STACK_SIZE = 1000;
155
156 /**
157 * Knuth's increments seem to work better than Incerpi-Sedgewick here.
158 * Possibly because the number of elems to sort is usually small, typically
159 * <= 20.
160 */
161 private static final int[] INCS = { 1, 4, 13, 40, 121, 364, 1093, 3280,
162 9841, 29524, 88573, 265720, 797161,
163 2391484 };
164
165 private static void hbMakeCodeLengths(final byte[] len, final int[] freq,
166 final Data dat, final int alphaSize,
167 final int maxLen) {
168 /*
169 * Nodes and heap entries run from 1. Entry 0 for both the heap and
170 * nodes is a sentinel.
171 */
172 final int[] heap = dat.heap;
173 final int[] weight = dat.weight;
174 final int[] parent = dat.parent;
175
176 for (int i = alphaSize; --i >= 0;) {
177 weight[i + 1] = (freq[i] == 0 ? 1 : freq[i]) << 8;
178 }
179
180 for (boolean tooLong = true; tooLong;) {
181 tooLong = false;
182
183 int nNodes = alphaSize;
184 int nHeap = 0;
185 heap[0] = 0;
186 weight[0] = 0;
187 parent[0] = -2;
188
189 for (int i = 1; i <= alphaSize; i++) {
190 parent[i] = -1;
191 nHeap++;
192 heap[nHeap] = i;
193
194 int zz = nHeap;
195 int tmp = heap[zz];
196 while (weight[tmp] < weight[heap[zz >> 1]]) {
197 heap[zz] = heap[zz >> 1];
198 zz >>= 1;
199 }
200 heap[zz] = tmp;
201 }
202
203 while (nHeap > 1) {
204 int n1 = heap[1];
205 heap[1] = heap[nHeap];
206 nHeap--;
207
208 int yy = 0;
209 int zz = 1;
210 int tmp = heap[1];
211
212 while (true) {
213 yy = zz << 1;
214
215 if (yy > nHeap) {
216 break;
217 }
218
219 if ((yy < nHeap)
220 && (weight[heap[yy + 1]] < weight[heap[yy]])) {
221 yy++;
222 }
223
224 if (weight[tmp] < weight[heap[yy]]) {
225 break;
226 }
227
228 heap[zz] = heap[yy];
229 zz = yy;
230 }
231
232 heap[zz] = tmp;
233
234 int n2 = heap[1];
235 heap[1] = heap[nHeap];
236 nHeap--;
237
238 yy = 0;
239 zz = 1;
240 tmp = heap[1];
241
242 while (true) {
243 yy = zz << 1;
244
245 if (yy > nHeap) {
246 break;
247 }
248
249 if ((yy < nHeap)
250 && (weight[heap[yy + 1]] < weight[heap[yy]])) {
251 yy++;
252 }
253
254 if (weight[tmp] < weight[heap[yy]]) {
255 break;
256 }
257
258 heap[zz] = heap[yy];
259 zz = yy;
260 }
261
262 heap[zz] = tmp;
263 nNodes++;
264 parent[n1] = parent[n2] = nNodes;
265
266 final int weight_n1 = weight[n1];
267 final int weight_n2 = weight[n2];
268 weight[nNodes] = ((weight_n1 & 0xffffff00)
269 + (weight_n2 & 0xffffff00))
270 | (1 + (((weight_n1 & 0x000000ff)
271 > (weight_n2 & 0x000000ff))
272 ? (weight_n1 & 0x000000ff)
273 : (weight_n2 & 0x000000ff)));
274
275 parent[nNodes] = -1;
276 nHeap++;
277 heap[nHeap] = nNodes;
278
279 tmp = 0;
280 zz = nHeap;
281 tmp = heap[zz];
282 final int weight_tmp = weight[tmp];
283 while (weight_tmp < weight[heap[zz >> 1]]) {
284 heap[zz] = heap[zz >> 1];
285 zz >>= 1;
286 }
287 heap[zz] = tmp;
288
289 }
290
291 for (int i = 1; i <= alphaSize; i++) {
292 int j = 0;
293 int k = i;
294
295 for (int parent_k; (parent_k = parent[k]) >= 0;) {
296 k = parent_k;
297 j++;
298 }
299
300 len[i - 1] = (byte) j;
301 if (j > maxLen) {
302 tooLong = true;
303 }
304 }
305
306 if (tooLong) {
307 for (int i = 1; i < alphaSize; i++) {
308 int j = weight[i] >> 8;
309 j = 1 + (j >> 1);
310 weight[i] = j << 8;
311 }
312 }
313 }
314 }
315
316 /**
317 * Index of the last char in the block, so the block size == last + 1.
318 */
319 private int last;
320
321 /**
322 * Index in fmap[] of original string after sorting.
323 */
324 private int origPtr;
325
326 /**
327 * Always: in the range 0 .. 9. The current block size is 100000 * this
328 * number.
329 */
330 private final int blockSize100k;
331
332 private boolean blockRandomised;
333
334 private int bsBuff;
335 private int bsLive;
336 private final CRC crc = new CRC();
337
338 private int nInUse;
339
340 private int nMTF;
341
342 /*
343 * Used when sorting. If too many long comparisons happen, we stop sorting,
344 * randomise the block slightly, and try again.
345 */
346 private int workDone;
347 private int workLimit;
348 private boolean firstAttempt;
349
350 private int currentChar = -1;
351 private int runLength = 0;
352
353 private int blockCRC;
354 private int combinedCRC;
355 private int allowableBlockSize;
356
357 /**
358 * All memory intensive stuff.
359 */
360 private Data data;
361
362 private OutputStream out;
363
364 /**
365 * Chooses a blocksize based on the given length of the data to compress.
366 *
367 * @return The blocksize, between {@link #MIN_BLOCKSIZE} and
368 * {@link #MAX_BLOCKSIZE} both inclusive. For a negative
369 * <tt>inputLength</tt> this method returns <tt>MAX_BLOCKSIZE</tt>
370 * always.
371 *
372 * @param inputLength
373 * The length of the data which will be compressed by
374 * <tt>CBZip2OutputStream</tt>.
375 */
376 public static int chooseBlockSize(long inputLength) {
377 return (inputLength > 0) ? (int) Math
378 .min((inputLength / 132000) + 1, 9) : MAX_BLOCKSIZE;
379 }
380
381 /**
382 * Constructs a new <tt>CBZip2OutputStream</tt> with a blocksize of 900k.
383 *
384 * @param out
385 * the destination stream.
386 *
387 * @throws IOException
388 * if an I/O error occurs in the specified stream.
389 * @throws NullPointerException
390 * if <code>out == null</code>.
391 */
392 public BZip2CompressorOutputStream(final OutputStream out)
393 throws IOException {
394 this(out, MAX_BLOCKSIZE);
395 }
396
397 /**
398 * Constructs a new <tt>CBZip2OutputStream</tt> with specified blocksize.
399 *
400 * @param out
401 * the destination stream.
402 * @param blockSize
403 * the blockSize as 100k units.
404 *
405 * @throws IOException
406 * if an I/O error occurs in the specified stream.
407 * @throws IllegalArgumentException
408 * if <code>(blockSize < 1) || (blockSize > 9)</code>.
409 * @throws NullPointerException
410 * if <code>out == null</code>.
411 *
412 * @see #MIN_BLOCKSIZE
413 * @see #MAX_BLOCKSIZE
414 */
415 public BZip2CompressorOutputStream(final OutputStream out,
416 final int blockSize)
417 throws IOException {
418 super();
419
420 if (blockSize < 1) {
421 throw new IllegalArgumentException("blockSize(" + blockSize
422 + ") < 1");
423 }
424 if (blockSize > 9) {
425 throw new IllegalArgumentException("blockSize(" + blockSize
426 + ") > 9");
427 }
428
429 this.blockSize100k = blockSize;
430 this.out = out;
431 init();
432 }
433
434 /** {@inheritDoc} */
435 @Override
436 public void write(final int b) throws IOException {
437 if (this.out != null) {
438 write0(b);
439 } else {
440 throw new IOException("closed");
441 }
442 }
443
444 private void writeRun() throws IOException {
445 final int lastShadow = this.last;
446
447 if (lastShadow < this.allowableBlockSize) {
448 final int currentCharShadow = this.currentChar;
449 final Data dataShadow = this.data;
450 dataShadow.inUse[currentCharShadow] = true;
451 final byte ch = (byte) currentCharShadow;
452
453 int runLengthShadow = this.runLength;
454 this.crc.updateCRC(currentCharShadow, runLengthShadow);
455
456 switch (runLengthShadow) {
457 case 1:
458 dataShadow.block[lastShadow + 2] = ch;
459 this.last = lastShadow + 1;
460 break;
461
462 case 2:
463 dataShadow.block[lastShadow + 2] = ch;
464 dataShadow.block[lastShadow + 3] = ch;
465 this.last = lastShadow + 2;
466 break;
467
468 case 3: {
469 final byte[] block = dataShadow.block;
470 block[lastShadow + 2] = ch;
471 block[lastShadow + 3] = ch;
472 block[lastShadow + 4] = ch;
473 this.last = lastShadow + 3;
474 }
475 break;
476
477 default: {
478 runLengthShadow -= 4;
479 dataShadow.inUse[runLengthShadow] = true;
480 final byte[] block = dataShadow.block;
481 block[lastShadow + 2] = ch;
482 block[lastShadow + 3] = ch;
483 block[lastShadow + 4] = ch;
484 block[lastShadow + 5] = ch;
485 block[lastShadow + 6] = (byte) runLengthShadow;
486 this.last = lastShadow + 5;
487 }
488 break;
489
490 }
491 } else {
492 endBlock();
493 initBlock();
494 writeRun();
495 }
496 }
497
498 /**
499 * Overriden to close the stream.
500 */
501 @Override
502 protected void finalize() throws Throwable {
503 finish();
504 super.finalize();
505 }
506
507
508 public void finish() throws IOException {
509 if (out != null) {
510 try {
511 if (this.runLength > 0) {
512 writeRun();
513 }
514 this.currentChar = -1;
515 endBlock();
516 endCompression();
517 } finally {
518 this.out = null;
519 this.data = null;
520 }
521 }
522 }
523
524 @Override
525 public void close() throws IOException {
526 if (out != null) {
527 OutputStream outShadow = this.out;
528 finish();
529 outShadow.close();
530 }
531 }
532
533 @Override
534 public void flush() throws IOException {
535 OutputStream outShadow = this.out;
536 if (outShadow != null) {
537 outShadow.flush();
538 }
539 }
540
541 /**
542 * Writes magic bytes like BZ on the first position of the stream
543 * and bytes indiciating the file-format, which is
544 * huffmanised, followed by a digit indicating blockSize100k.
545 * @throws IOException if the magic bytes could not been written
546 */
547 private void init() throws IOException {
548 bsPutUByte('B');
549 bsPutUByte('Z');
550
551 this.data = new Data(this.blockSize100k);
552
553 // huffmanised magic bytes
554 bsPutUByte('h');
555 bsPutUByte('0' + this.blockSize100k);
556
557 this.combinedCRC = 0;
558 initBlock();
559 }
560
561 private void initBlock() {
562 // blockNo++;
563 this.crc.initialiseCRC();
564 this.last = -1;
565 // ch = 0;
566
567 boolean[] inUse = this.data.inUse;
568 for (int i = 256; --i >= 0;) {
569 inUse[i] = false;
570 }
571
572 /* 20 is just a paranoia constant */
573 this.allowableBlockSize = (this.blockSize100k * BZip2Constants.BASEBLOCKSIZE) - 20;
574 }
575
576 private void endBlock() throws IOException {
577 this.blockCRC = this.crc.getFinalCRC();
578 this.combinedCRC = (this.combinedCRC << 1) | (this.combinedCRC >>> 31);
579 this.combinedCRC ^= this.blockCRC;
580
581 // empty block at end of file
582 if (this.last == -1) {
583 return;
584 }
585
586 /* sort the block and establish posn of original string */
587 blockSort();
588
589 /*
590 * A 6-byte block header, the value chosen arbitrarily as 0x314159265359
591 * :-). A 32 bit value does not really give a strong enough guarantee
592 * that the value will not appear by chance in the compressed
593 * datastream. Worst-case probability of this event, for a 900k block,
594 * is about 2.0e-3 for 32 bits, 1.0e-5 for 40 bits and 4.0e-8 for 48
595 * bits. For a compressed file of size 100Gb -- about 100000 blocks --
596 * only a 48-bit marker will do. NB: normal compression/ decompression
597 * donot rely on these statistical properties. They are only important
598 * when trying to recover blocks from damaged files.
599 */
600 bsPutUByte(0x31);
601 bsPutUByte(0x41);
602 bsPutUByte(0x59);
603 bsPutUByte(0x26);
604 bsPutUByte(0x53);
605 bsPutUByte(0x59);
606
607 /* Now the block's CRC, so it is in a known place. */
608 bsPutInt(this.blockCRC);
609
610 /* Now a single bit indicating randomisation. */
611 if (this.blockRandomised) {
612 bsW(1, 1);
613 } else {
614 bsW(1, 0);
615 }
616
617 /* Finally, block's contents proper. */
618 moveToFrontCodeAndSend();
619 }
620
621 private void endCompression() throws IOException {
622 /*
623 * Now another magic 48-bit number, 0x177245385090, to indicate the end
624 * of the last block. (sqrt(pi), if you want to know. I did want to use
625 * e, but it contains too much repetition -- 27 18 28 18 28 46 -- for me
626 * to feel statistically comfortable. Call me paranoid.)
627 */
628 bsPutUByte(0x17);
629 bsPutUByte(0x72);
630 bsPutUByte(0x45);
631 bsPutUByte(0x38);
632 bsPutUByte(0x50);
633 bsPutUByte(0x90);
634
635 bsPutInt(this.combinedCRC);
636 bsFinishedWithStream();
637 }
638
639 /**
640 * Returns the blocksize parameter specified at construction time.
641 */
642 public final int getBlockSize() {
643 return this.blockSize100k;
644 }
645
646 @Override
647 public void write(final byte[] buf, int offs, final int len)
648 throws IOException {
649 if (offs < 0) {
650 throw new IndexOutOfBoundsException("offs(" + offs + ") < 0.");
651 }
652 if (len < 0) {
653 throw new IndexOutOfBoundsException("len(" + len + ") < 0.");
654 }
655 if (offs + len > buf.length) {
656 throw new IndexOutOfBoundsException("offs(" + offs + ") + len("
657 + len + ") > buf.length("
658 + buf.length + ").");
659 }
660 if (this.out == null) {
661 throw new IOException("stream closed");
662 }
663
664 for (int hi = offs + len; offs < hi;) {
665 write0(buf[offs++]);
666 }
667 }
668
669 private void write0(int b) throws IOException {
670 if (this.currentChar != -1) {
671 b &= 0xff;
672 if (this.currentChar == b) {
673 if (++this.runLength > 254) {
674 writeRun();
675 this.currentChar = -1;
676 this.runLength = 0;
677 }
678 // else nothing to do
679 } else {
680 writeRun();
681 this.runLength = 1;
682 this.currentChar = b;
683 }
684 } else {
685 this.currentChar = b & 0xff;
686 this.runLength++;
687 }
688 }
689
690 private static void hbAssignCodes(final int[] code, final byte[] length,
691 final int minLen, final int maxLen,
692 final int alphaSize) {
693 int vec = 0;
694 for (int n = minLen; n <= maxLen; n++) {
695 for (int i = 0; i < alphaSize; i++) {
696 if ((length[i] & 0xff) == n) {
697 code[i] = vec;
698 vec++;
699 }
700 }
701 vec <<= 1;
702 }
703 }
704
705 private void bsFinishedWithStream() throws IOException {
706 while (this.bsLive > 0) {
707 int ch = this.bsBuff >> 24;
708 this.out.write(ch); // write 8-bit
709 this.bsBuff <<= 8;
710 this.bsLive -= 8;
711 }
712 }
713
714 private void bsW(final int n, final int v) throws IOException {
715 final OutputStream outShadow = this.out;
716 int bsLiveShadow = this.bsLive;
717 int bsBuffShadow = this.bsBuff;
718
719 while (bsLiveShadow >= 8) {
720 outShadow.write(bsBuffShadow >> 24); // write 8-bit
721 bsBuffShadow <<= 8;
722 bsLiveShadow -= 8;
723 }
724
725 this.bsBuff = bsBuffShadow | (v << (32 - bsLiveShadow - n));
726 this.bsLive = bsLiveShadow + n;
727 }
728
729 private void bsPutUByte(final int c) throws IOException {
730 bsW(8, c);
731 }
732
733 private void bsPutInt(final int u) throws IOException {
734 bsW(8, (u >> 24) & 0xff);
735 bsW(8, (u >> 16) & 0xff);
736 bsW(8, (u >> 8) & 0xff);
737 bsW(8, u & 0xff);
738 }
739
740 private void sendMTFValues() throws IOException {
741 final byte[][] len = this.data.sendMTFValues_len;
742 final int alphaSize = this.nInUse + 2;
743
744 for (int t = N_GROUPS; --t >= 0;) {
745 byte[] len_t = len[t];
746 for (int v = alphaSize; --v >= 0;) {
747 len_t[v] = GREATER_ICOST;
748 }
749 }
750
751 /* Decide how many coding tables to use */
752 // assert (this.nMTF > 0) : this.nMTF;
753 final int nGroups = (this.nMTF < 200) ? 2 : (this.nMTF < 600) ? 3
754 : (this.nMTF < 1200) ? 4 : (this.nMTF < 2400) ? 5 : 6;
755
756 /* Generate an initial set of coding tables */
757 sendMTFValues0(nGroups, alphaSize);
758
759 /*
760 * Iterate up to N_ITERS times to improve the tables.
761 */
762 final int nSelectors = sendMTFValues1(nGroups, alphaSize);
763
764 /* Compute MTF values for the selectors. */
765 sendMTFValues2(nGroups, nSelectors);
766
767 /* Assign actual codes for the tables. */
768 sendMTFValues3(nGroups, alphaSize);
769
770 /* Transmit the mapping table. */
771 sendMTFValues4();
772
773 /* Now the selectors. */
774 sendMTFValues5(nGroups, nSelectors);
775
776 /* Now the coding tables. */
777 sendMTFValues6(nGroups, alphaSize);
778
779 /* And finally, the block data proper */
780 sendMTFValues7();
781 }
782
783 private void sendMTFValues0(final int nGroups, final int alphaSize) {
784 final byte[][] len = this.data.sendMTFValues_len;
785 final int[] mtfFreq = this.data.mtfFreq;
786
787 int remF = this.nMTF;
788 int gs = 0;
789
790 for (int nPart = nGroups; nPart > 0; nPart--) {
791 final int tFreq = remF / nPart;
792 int ge = gs - 1;
793 int aFreq = 0;
794
795 for (final int a = alphaSize - 1; (aFreq < tFreq) && (ge < a);) {
796 aFreq += mtfFreq[++ge];
797 }
798
799 if ((ge > gs) && (nPart != nGroups) && (nPart != 1)
800 && (((nGroups - nPart) & 1) != 0)) {
801 aFreq -= mtfFreq[ge--];
802 }
803
804 final byte[] len_np = len[nPart - 1];
805 for (int v = alphaSize; --v >= 0;) {
806 if ((v >= gs) && (v <= ge)) {
807 len_np[v] = LESSER_ICOST;
808 } else {
809 len_np[v] = GREATER_ICOST;
810 }
811 }
812
813 gs = ge + 1;
814 remF -= aFreq;
815 }
816 }
817
818 private int sendMTFValues1(final int nGroups, final int alphaSize) {
819 final Data dataShadow = this.data;
820 final int[][] rfreq = dataShadow.sendMTFValues_rfreq;
821 final int[] fave = dataShadow.sendMTFValues_fave;
822 final short[] cost = dataShadow.sendMTFValues_cost;
823 final char[] sfmap = dataShadow.sfmap;
824 final byte[] selector = dataShadow.selector;
825 final byte[][] len = dataShadow.sendMTFValues_len;
826 final byte[] len_0 = len[0];
827 final byte[] len_1 = len[1];
828 final byte[] len_2 = len[2];
829 final byte[] len_3 = len[3];
830 final byte[] len_4 = len[4];
831 final byte[] len_5 = len[5];
832 final int nMTFShadow = this.nMTF;
833
834 int nSelectors = 0;
835
836 for (int iter = 0; iter < N_ITERS; iter++) {
837 for (int t = nGroups; --t >= 0;) {
838 fave[t] = 0;
839 int[] rfreqt = rfreq[t];
840 for (int i = alphaSize; --i >= 0;) {
841 rfreqt[i] = 0;
842 }
843 }
844
845 nSelectors = 0;
846
847 for (int gs = 0; gs < this.nMTF;) {
848 /* Set group start & end marks. */
849
850 /*
851 * Calculate the cost of this group as coded by each of the
852 * coding tables.
853 */
854
855 final int ge = Math.min(gs + G_SIZE - 1, nMTFShadow - 1);
856
857 if (nGroups == N_GROUPS) {
858 // unrolled version of the else-block
859
860 short cost0 = 0;
861 short cost1 = 0;
862 short cost2 = 0;
863 short cost3 = 0;
864 short cost4 = 0;
865 short cost5 = 0;
866
867 for (int i = gs; i <= ge; i++) {
868 final int icv = sfmap[i];
869 cost0 += len_0[icv] & 0xff;
870 cost1 += len_1[icv] & 0xff;
871 cost2 += len_2[icv] & 0xff;
872 cost3 += len_3[icv] & 0xff;
873 cost4 += len_4[icv] & 0xff;
874 cost5 += len_5[icv] & 0xff;
875 }
876
877 cost[0] = cost0;
878 cost[1] = cost1;
879 cost[2] = cost2;
880 cost[3] = cost3;
881 cost[4] = cost4;
882 cost[5] = cost5;
883
884 } else {
885 for (int t = nGroups; --t >= 0;) {
886 cost[t] = 0;
887 }
888
889 for (int i = gs; i <= ge; i++) {
890 final int icv = sfmap[i];
891 for (int t = nGroups; --t >= 0;) {
892 cost[t] += len[t][icv] & 0xff;
893 }
894 }
895 }
896
897 /*
898 * Find the coding table which is best for this group, and
899 * record its identity in the selector table.
900 */
901 int bt = -1;
902 for (int t = nGroups, bc = 999999999; --t >= 0;) {
903 final int cost_t = cost[t];
904 if (cost_t < bc) {
905 bc = cost_t;
906 bt = t;
907 }
908 }
909
910 fave[bt]++;
911 selector[nSelectors] = (byte) bt;
912 nSelectors++;
913
914 /*
915 * Increment the symbol frequencies for the selected table.
916 */
917 final int[] rfreq_bt = rfreq[bt];
918 for (int i = gs; i <= ge; i++) {
919 rfreq_bt[sfmap[i]]++;
920 }
921
922 gs = ge + 1;
923 }
924
925 /*
926 * Recompute the tables based on the accumulated frequencies.
927 */
928 for (int t = 0; t < nGroups; t++) {
929 hbMakeCodeLengths(len[t], rfreq[t], this.data, alphaSize, 20);
930 }
931 }
932
933 return nSelectors;
934 }
935
936 private void sendMTFValues2(final int nGroups, final int nSelectors) {
937 // assert (nGroups < 8) : nGroups;
938
939 final Data dataShadow = this.data;
940 byte[] pos = dataShadow.sendMTFValues2_pos;
941
942 for (int i = nGroups; --i >= 0;) {
943 pos[i] = (byte) i;
944 }
945
946 for (int i = 0; i < nSelectors; i++) {
947 final byte ll_i = dataShadow.selector[i];
948 byte tmp = pos[0];
949 int j = 0;
950
951 while (ll_i != tmp) {
952 j++;
953 byte tmp2 = tmp;
954 tmp = pos[j];
955 pos[j] = tmp2;
956 }
957
958 pos[0] = tmp;
959 dataShadow.selectorMtf[i] = (byte) j;
960 }
961 }
962
963 private void sendMTFValues3(final int nGroups, final int alphaSize) {
964 int[][] code = this.data.sendMTFValues_code;
965 byte[][] len = this.data.sendMTFValues_len;
966
967 for (int t = 0; t < nGroups; t++) {
968 int minLen = 32;
969 int maxLen = 0;
970 final byte[] len_t = len[t];
971 for (int i = alphaSize; --i >= 0;) {
972 final int l = len_t[i] & 0xff;
973 if (l > maxLen) {
974 maxLen = l;
975 }
976 if (l < minLen) {
977 minLen = l;
978 }
979 }
980
981 // assert (maxLen <= 20) : maxLen;
982 // assert (minLen >= 1) : minLen;
983
984 hbAssignCodes(code[t], len[t], minLen, maxLen, alphaSize);
985 }
986 }
987
988 private void sendMTFValues4() throws IOException {
989 final boolean[] inUse = this.data.inUse;
990 final boolean[] inUse16 = this.data.sentMTFValues4_inUse16;
991
992 for (int i = 16; --i >= 0;) {
993 inUse16[i] = false;
994 final int i16 = i * 16;
995 for (int j = 16; --j >= 0;) {
996 if (inUse[i16 + j]) {
997 inUse16[i] = true;
998 }
999 }
1000 }
1001
1002 for (int i = 0; i < 16; i++) {
1003 bsW(1, inUse16[i] ? 1 : 0);
1004 }
1005
1006 final OutputStream outShadow = this.out;
1007 int bsLiveShadow = this.bsLive;
1008 int bsBuffShadow = this.bsBuff;
1009
1010 for (int i = 0; i < 16; i++) {
1011 if (inUse16[i]) {
1012 final int i16 = i * 16;
1013 for (int j = 0; j < 16; j++) {
1014 // inlined: bsW(1, inUse[i16 + j] ? 1 : 0);
1015 while (bsLiveShadow >= 8) {
1016 outShadow.write(bsBuffShadow >> 24); // write 8-bit
1017 bsBuffShadow <<= 8;
1018 bsLiveShadow -= 8;
1019 }
1020 if (inUse[i16 + j]) {
1021 bsBuffShadow |= 1 << (32 - bsLiveShadow - 1);
1022 }
1023 bsLiveShadow++;
1024 }
1025 }
1026 }
1027
1028 this.bsBuff = bsBuffShadow;
1029 this.bsLive = bsLiveShadow;
1030 }
1031
1032 private void sendMTFValues5(final int nGroups, final int nSelectors)
1033 throws IOException {
1034 bsW(3, nGroups);
1035 bsW(15, nSelectors);
1036
1037 final OutputStream outShadow = this.out;
1038 final byte[] selectorMtf = this.data.selectorMtf;
1039
1040 int bsLiveShadow = this.bsLive;
1041 int bsBuffShadow = this.bsBuff;
1042
1043 for (int i = 0; i < nSelectors; i++) {
1044 for (int j = 0, hj = selectorMtf[i] & 0xff; j < hj; j++) {
1045 // inlined: bsW(1, 1);
1046 while (bsLiveShadow >= 8) {
1047 outShadow.write(bsBuffShadow >> 24);
1048 bsBuffShadow <<= 8;
1049 bsLiveShadow -= 8;
1050 }
1051 bsBuffShadow |= 1 << (32 - bsLiveShadow - 1);
1052 bsLiveShadow++;
1053 }
1054
1055 // inlined: bsW(1, 0);
1056 while (bsLiveShadow >= 8) {
1057 outShadow.write(bsBuffShadow >> 24);
1058 bsBuffShadow <<= 8;
1059 bsLiveShadow -= 8;
1060 }
1061 // bsBuffShadow |= 0 << (32 - bsLiveShadow - 1);
1062 bsLiveShadow++;
1063 }
1064
1065 this.bsBuff = bsBuffShadow;
1066 this.bsLive = bsLiveShadow;
1067 }
1068
1069 private void sendMTFValues6(final int nGroups, final int alphaSize)
1070 throws IOException {
1071 final byte[][] len = this.data.sendMTFValues_len;
1072 final OutputStream outShadow = this.out;
1073
1074 int bsLiveShadow = this.bsLive;
1075 int bsBuffShadow = this.bsBuff;
1076
1077 for (int t = 0; t < nGroups; t++) {
1078 byte[] len_t = len[t];
1079 int curr = len_t[0] & 0xff;
1080
1081 // inlined: bsW(5, curr);
1082 while (bsLiveShadow >= 8) {
1083 outShadow.write(bsBuffShadow >> 24); // write 8-bit
1084 bsBuffShadow <<= 8;
1085 bsLiveShadow -= 8;
1086 }
1087 bsBuffShadow |= curr << (32 - bsLiveShadow - 5);
1088 bsLiveShadow += 5;
1089
1090 for (int i = 0; i < alphaSize; i++) {
1091 int lti = len_t[i] & 0xff;
1092 while (curr < lti) {
1093 // inlined: bsW(2, 2);
1094 while (bsLiveShadow >= 8) {
1095 outShadow.write(bsBuffShadow >> 24); // write 8-bit
1096 bsBuffShadow <<= 8;
1097 bsLiveShadow -= 8;
1098 }
1099 bsBuffShadow |= 2 << (32 - bsLiveShadow - 2);
1100 bsLiveShadow += 2;
1101
1102 curr++; /* 10 */
1103 }
1104
1105 while (curr > lti) {
1106 // inlined: bsW(2, 3);
1107 while (bsLiveShadow >= 8) {
1108 outShadow.write(bsBuffShadow >> 24); // write 8-bit
1109 bsBuffShadow <<= 8;
1110 bsLiveShadow -= 8;
1111 }
1112 bsBuffShadow |= 3 << (32 - bsLiveShadow - 2);
1113 bsLiveShadow += 2;
1114
1115 curr--; /* 11 */
1116 }
1117
1118 // inlined: bsW(1, 0);
1119 while (bsLiveShadow >= 8) {
1120 outShadow.write(bsBuffShadow >> 24); // write 8-bit
1121 bsBuffShadow <<= 8;
1122 bsLiveShadow -= 8;
1123 }
1124 // bsBuffShadow |= 0 << (32 - bsLiveShadow - 1);
1125 bsLiveShadow++;
1126 }
1127 }
1128
1129 this.bsBuff = bsBuffShadow;
1130 this.bsLive = bsLiveShadow;
1131 }
1132
1133 private void sendMTFValues7() throws IOException {
1134 final Data dataShadow = this.data;
1135 final byte[][] len = dataShadow.sendMTFValues_len;
1136 final int[][] code = dataShadow.sendMTFValues_code;
1137 final OutputStream outShadow = this.out;
1138 final byte[] selector = dataShadow.selector;
1139 final char[] sfmap = dataShadow.sfmap;
1140 final int nMTFShadow = this.nMTF;
1141
1142 int selCtr = 0;
1143
1144 int bsLiveShadow = this.bsLive;
1145 int bsBuffShadow = this.bsBuff;
1146
1147 for (int gs = 0; gs < nMTFShadow;) {
1148 final int ge = Math.min(gs + G_SIZE - 1, nMTFShadow - 1);
1149 final int selector_selCtr = selector[selCtr] & 0xff;
1150 final int[] code_selCtr = code[selector_selCtr];
1151 final byte[] len_selCtr = len[selector_selCtr];
1152
1153 while (gs <= ge) {
1154 final int sfmap_i = sfmap[gs];
1155
1156 //
1157 // inlined: bsW(len_selCtr[sfmap_i] & 0xff,
1158 // code_selCtr[sfmap_i]);
1159 //
1160 while (bsLiveShadow >= 8) {
1161 outShadow.write(bsBuffShadow >> 24);
1162 bsBuffShadow <<= 8;
1163 bsLiveShadow -= 8;
1164 }
1165 final int n = len_selCtr[sfmap_i] & 0xFF;
1166 bsBuffShadow |= code_selCtr[sfmap_i] << (32 - bsLiveShadow - n);
1167 bsLiveShadow += n;
1168
1169 gs++;
1170 }
1171
1172 gs = ge + 1;
1173 selCtr++;
1174 }
1175
1176 this.bsBuff = bsBuffShadow;
1177 this.bsLive = bsLiveShadow;
1178 }
1179
1180 private void moveToFrontCodeAndSend() throws IOException {
1181 bsW(24, this.origPtr);
1182 generateMTFValues();
1183 sendMTFValues();
1184 }
1185
1186 /**
1187 * This is the most hammered method of this class.
1188 *
1189 * <p>
1190 * This is the version using unrolled loops. Normally I never use such ones
1191 * in Java code. The unrolling has shown a noticable performance improvement
1192 * on JRE 1.4.2 (Linux i586 / HotSpot Client). Of course it depends on the
1193 * JIT compiler of the vm.
1194 * </p>
1195 */
1196 private boolean mainSimpleSort(final Data dataShadow, final int lo,
1197 final int hi, final int d) {
1198 final int bigN = hi - lo + 1;
1199 if (bigN < 2) {
1200 return this.firstAttempt && (this.workDone > this.workLimit);
1201 }
1202
1203 int hp = 0;
1204 while (INCS[hp] < bigN) {
1205 hp++;
1206 }
1207
1208 final int[] fmap = dataShadow.fmap;
1209 final char[] quadrant = dataShadow.quadrant;
1210 final byte[] block = dataShadow.block;
1211 final int lastShadow = this.last;
1212 final int lastPlus1 = lastShadow + 1;
1213 final boolean firstAttemptShadow = this.firstAttempt;
1214 final int workLimitShadow = this.workLimit;
1215 int workDoneShadow = this.workDone;
1216
1217 // Following block contains unrolled code which could be shortened by
1218 // coding it in additional loops.
1219
1220 HP: while (--hp >= 0) {
1221 final int h = INCS[hp];
1222 final int mj = lo + h - 1;
1223
1224 for (int i = lo + h; i <= hi;) {
1225 // copy
1226 for (int k = 3; (i <= hi) && (--k >= 0); i++) {
1227 final int v = fmap[i];
1228 final int vd = v + d;
1229 int j = i;
1230
1231 // for (int a;
1232 // (j > mj) && mainGtU((a = fmap[j - h]) + d, vd,
1233 // block, quadrant, lastShadow);
1234 // j -= h) {
1235 // fmap[j] = a;
1236 // }
1237 //
1238 // unrolled version:
1239
1240 // start inline mainGTU
1241 boolean onceRunned = false;
1242 int a = 0;
1243
1244 HAMMER: while (true) {
1245 if (onceRunned) {
1246 fmap[j] = a;
1247 if ((j -= h) <= mj) {
1248 break HAMMER;
1249 }
1250 } else {
1251 onceRunned = true;
1252 }
1253
1254 a = fmap[j - h];
1255 int i1 = a + d;
1256 int i2 = vd;
1257
1258 // following could be done in a loop, but
1259 // unrolled it for performance:
1260 if (block[i1 + 1] == block[i2 + 1]) {
1261 if (block[i1 + 2] == block[i2 + 2]) {
1262 if (block[i1 + 3] == block[i2 + 3]) {
1263 if (block[i1 + 4] == block[i2 + 4]) {
1264 if (block[i1 + 5] == block[i2 + 5]) {
1265 if (block[(i1 += 6)] == block[(i2 += 6)]) {
1266 int x = lastShadow;
1267 X: while (x > 0) {
1268 x -= 4;
1269
1270 if (block[i1 + 1] == block[i2 + 1]) {
1271 if (quadrant[i1] == quadrant[i2]) {
1272 if (block[i1 + 2] == block[i2 + 2]) {
1273 if (quadrant[i1 + 1] == quadrant[i2 + 1]) {
1274 if (block[i1 + 3] == block[i2 + 3]) {
1275 if (quadrant[i1 + 2] == quadrant[i2 + 2]) {
1276 if (block[i1 + 4] == block[i2 + 4]) {
1277 if (quadrant[i1 + 3] == quadrant[i2 + 3]) {
1278 if ((i1 += 4) >= lastPlus1) {
1279 i1 -= lastPlus1;
1280 }
1281 if ((i2 += 4) >= lastPlus1) {
1282 i2 -= lastPlus1;
1283 }
1284 workDoneShadow++;
1285 continue X;
1286 } else if ((quadrant[i1 + 3] > quadrant[i2 + 3])) {
1287 continue HAMMER;
1288 } else {
1289 break HAMMER;
1290 }
1291 } else if ((block[i1 + 4] & 0xff) > (block[i2 + 4] & 0xff)) {
1292 continue HAMMER;
1293 } else {
1294 break HAMMER;
1295 }
1296 } else if ((quadrant[i1 + 2] > quadrant[i2 + 2])) {
1297 continue HAMMER;
1298 } else {
1299 break HAMMER;
1300 }
1301 } else if ((block[i1 + 3] & 0xff) > (block[i2 + 3] & 0xff)) {
1302 continue HAMMER;
1303 } else {
1304 break HAMMER;
1305 }
1306 } else if ((quadrant[i1 + 1] > quadrant[i2 + 1])) {
1307 continue HAMMER;
1308 } else {
1309 break HAMMER;
1310 }
1311 } else if ((block[i1 + 2] & 0xff) > (block[i2 + 2] & 0xff)) {
1312 continue HAMMER;
1313 } else {
1314 break HAMMER;
1315 }
1316 } else if ((quadrant[i1] > quadrant[i2])) {
1317 continue HAMMER;
1318 } else {
1319 break HAMMER;
1320 }
1321 } else if ((block[i1 + 1] & 0xff) > (block[i2 + 1] & 0xff)) {
1322 continue HAMMER;
1323 } else {
1324 break HAMMER;
1325 }
1326
1327 }
1328 break HAMMER;
1329 } // while x > 0
1330 else {
1331 if ((block[i1] & 0xff) > (block[i2] & 0xff)) {
1332 continue HAMMER;
1333 } else {
1334 break HAMMER;
1335 }
1336 }
1337 } else if ((block[i1 + 5] & 0xff) > (block[i2 + 5] & 0xff)) {
1338 continue HAMMER;
1339 } else {
1340 break HAMMER;
1341 }
1342 } else if ((block[i1 + 4] & 0xff) > (block[i2 + 4] & 0xff)) {
1343 continue HAMMER;
1344 } else {
1345 break HAMMER;
1346 }
1347 } else if ((block[i1 + 3] & 0xff) > (block[i2 + 3] & 0xff)) {
1348 continue HAMMER;
1349 } else {
1350 break HAMMER;
1351 }
1352 } else if ((block[i1 + 2] & 0xff) > (block[i2 + 2] & 0xff)) {
1353 continue HAMMER;
1354 } else {
1355 break HAMMER;
1356 }
1357 } else if ((block[i1 + 1] & 0xff) > (block[i2 + 1] & 0xff)) {
1358 continue HAMMER;
1359 } else {
1360 break HAMMER;
1361 }
1362
1363 } // HAMMER
1364 // end inline mainGTU
1365
1366 fmap[j] = v;
1367 }
1368
1369 if (firstAttemptShadow && (i <= hi)
1370 && (workDoneShadow > workLimitShadow)) {
1371 break HP;
1372 }
1373 }
1374 }
1375
1376 this.workDone = workDoneShadow;
1377 return firstAttemptShadow && (workDoneShadow > workLimitShadow);
1378 }
1379
1380 private static void vswap(int[] fmap, int p1, int p2, int n) {
1381 n += p1;
1382 while (p1 < n) {
1383 int t = fmap[p1];
1384 fmap[p1++] = fmap[p2];
1385 fmap[p2++] = t;
1386 }
1387 }
1388
1389 private static byte med3(byte a, byte b, byte c) {
1390 return (a < b) ? (b < c ? b : a < c ? c : a) : (b > c ? b : a > c ? c
1391 : a);
1392 }
1393
1394 private void blockSort() {
1395 this.workLimit = WORK_FACTOR * this.last;
1396 this.workDone = 0;
1397 this.blockRandomised = false;
1398 this.firstAttempt = true;
1399 mainSort();
1400
1401 if (this.firstAttempt && (this.workDone > this.workLimit)) {
1402 randomiseBlock();
1403 this.workLimit = this.workDone = 0;
1404 this.firstAttempt = false;
1405 mainSort();
1406 }
1407
1408 int[] fmap = this.data.fmap;
1409 this.origPtr = -1;
1410 for (int i = 0, lastShadow = this.last; i <= lastShadow; i++) {
1411 if (fmap[i] == 0) {
1412 this.origPtr = i;
1413 break;
1414 }
1415 }
1416
1417 // assert (this.origPtr != -1) : this.origPtr;
1418 }
1419
1420 /**
1421 * Method "mainQSort3", file "blocksort.c", BZip2 1.0.2
1422 */
1423 private void mainQSort3(final Data dataShadow, final int loSt,
1424 final int hiSt, final int dSt) {
1425 final int[] stack_ll = dataShadow.stack_ll;
1426 final int[] stack_hh = dataShadow.stack_hh;
1427 final int[] stack_dd = dataShadow.stack_dd;
1428 final int[] fmap = dataShadow.fmap;
1429 final byte[] block = dataShadow.block;
1430
1431 stack_ll[0] = loSt;
1432 stack_hh[0] = hiSt;
1433 stack_dd[0] = dSt;
1434
1435 for (int sp = 1; --sp >= 0;) {
1436 final int lo = stack_ll[sp];
1437 final int hi = stack_hh[sp];
1438 final int d = stack_dd[sp];
1439
1440 if ((hi - lo < SMALL_THRESH) || (d > DEPTH_THRESH)) {
1441 if (mainSimpleSort(dataShadow, lo, hi, d)) {
1442 return;
1443 }
1444 } else {
1445 final int d1 = d + 1;
1446 final int med = med3(block[fmap[lo] + d1],
1447 block[fmap[hi] + d1], block[fmap[(lo + hi) >>> 1] + d1]) & 0xff;
1448
1449 int unLo = lo;
1450 int unHi = hi;
1451 int ltLo = lo;
1452 int gtHi = hi;
1453
1454 while (true) {
1455 while (unLo <= unHi) {
1456 final int n = (block[fmap[unLo] + d1] & 0xff)
1457 - med;
1458 if (n == 0) {
1459 final int temp = fmap[unLo];
1460 fmap[unLo++] = fmap[ltLo];
1461 fmap[ltLo++] = temp;
1462 } else if (n < 0) {
1463 unLo++;
1464 } else {
1465 break;
1466 }
1467 }
1468
1469 while (unLo <= unHi) {
1470 final int n = (block[fmap[unHi] + d1] & 0xff)
1471 - med;
1472 if (n == 0) {
1473 final int temp = fmap[unHi];
1474 fmap[unHi--] = fmap[gtHi];
1475 fmap[gtHi--] = temp;
1476 } else if (n > 0) {
1477 unHi--;
1478 } else {
1479 break;
1480 }
1481 }
1482
1483 if (unLo <= unHi) {
1484 final int temp = fmap[unLo];
1485 fmap[unLo++] = fmap[unHi];
1486 fmap[unHi--] = temp;
1487 } else {
1488 break;
1489 }
1490 }
1491
1492 if (gtHi < ltLo) {
1493 stack_ll[sp] = lo;
1494 stack_hh[sp] = hi;
1495 stack_dd[sp] = d1;
1496 sp++;
1497 } else {
1498 int n = ((ltLo - lo) < (unLo - ltLo)) ? (ltLo - lo)
1499 : (unLo - ltLo);
1500 vswap(fmap, lo, unLo - n, n);
1501 int m = ((hi - gtHi) < (gtHi - unHi)) ? (hi - gtHi)
1502 : (gtHi - unHi);
1503 vswap(fmap, unLo, hi - m + 1, m);
1504
1505 n = lo + unLo - ltLo - 1;
1506 m = hi - (gtHi - unHi) + 1;
1507
1508 stack_ll[sp] = lo;
1509 stack_hh[sp] = n;
1510 stack_dd[sp] = d;
1511 sp++;
1512
1513 stack_ll[sp] = n + 1;
1514 stack_hh[sp] = m - 1;
1515 stack_dd[sp] = d1;
1516 sp++;
1517
1518 stack_ll[sp] = m;
1519 stack_hh[sp] = hi;
1520 stack_dd[sp] = d;
1521 sp++;
1522 }
1523 }
1524 }
1525 }
1526
1527 private void mainSort() {
1528 final Data dataShadow = this.data;
1529 final int[] runningOrder = dataShadow.mainSort_runningOrder;
1530 final int[] copy = dataShadow.mainSort_copy;
1531 final boolean[] bigDone = dataShadow.mainSort_bigDone;
1532 final int[] ftab = dataShadow.ftab;
1533 final byte[] block = dataShadow.block;
1534 final int[] fmap = dataShadow.fmap;
1535 final char[] quadrant = dataShadow.quadrant;
1536 final int lastShadow = this.last;
1537 final int workLimitShadow = this.workLimit;
1538 final boolean firstAttemptShadow = this.firstAttempt;
1539
1540 // Set up the 2-byte frequency table
1541 for (int i = 65537; --i >= 0;) {
1542 ftab[i] = 0;
1543 }
1544
1545 /*
1546 * In the various block-sized structures, live data runs from 0 to
1547 * last+NUM_OVERSHOOT_BYTES inclusive. First, set up the overshoot area
1548 * for block.
1549 */
1550 for (int i = 0; i < NUM_OVERSHOOT_BYTES; i++) {
1551 block[lastShadow + i + 2] = block[(i % (lastShadow + 1)) + 1];
1552 }
1553 for (int i = lastShadow + NUM_OVERSHOOT_BYTES +1; --i >= 0;) {
1554 quadrant[i] = 0;
1555 }
1556 block[0] = block[lastShadow + 1];
1557
1558 // Complete the initial radix sort:
1559
1560 int c1 = block[0] & 0xff;
1561 for (int i = 0; i <= lastShadow; i++) {
1562 final int c2 = block[i + 1] & 0xff;
1563 ftab[(c1 << 8) + c2]++;
1564 c1 = c2;
1565 }
1566
1567 for (int i = 1; i <= 65536; i++) {
1568 ftab[i] += ftab[i - 1];
1569 }
1570
1571 c1 = block[1] & 0xff;
1572 for (int i = 0; i < lastShadow; i++) {
1573 final int c2 = block[i + 2] & 0xff;
1574 fmap[--ftab[(c1 << 8) + c2]] = i;
1575 c1 = c2;
1576 }
1577
1578 fmap[--ftab[((block[lastShadow + 1] & 0xff) << 8) + (block[1] & 0xff)]] = lastShadow;
1579
1580 /*
1581 * Now ftab contains the first loc of every small bucket. Calculate the
1582 * running order, from smallest to largest big bucket.
1583 */
1584 for (int i = 256; --i >= 0;) {
1585 bigDone[i] = false;
1586 runningOrder[i] = i;
1587 }
1588
1589 for (int h = 364; h != 1;) {
1590 h /= 3;
1591 for (int i = h; i <= 255; i++) {
1592 final int vv = runningOrder[i];
1593 final int a = ftab[(vv + 1) << 8] - ftab[vv << 8];
1594 final int b = h - 1;
1595 int j = i;
1596 for (int ro = runningOrder[j - h]; (ftab[(ro + 1) << 8] - ftab[ro << 8]) > a; ro = runningOrder[j
1597 - h]) {
1598 runningOrder[j] = ro;
1599 j -= h;
1600 if (j <= b) {
1601 break;
1602 }
1603 }
1604 runningOrder[j] = vv;
1605 }
1606 }
1607
1608 /*
1609 * The main sorting loop.
1610 */
1611 for (int i = 0; i <= 255; i++) {
1612 /*
1613 * Process big buckets, starting with the least full.
1614 */
1615 final int ss = runningOrder[i];
1616
1617 // Step 1:
1618 /*
1619 * Complete the big bucket [ss] by quicksorting any unsorted small
1620 * buckets [ss, j]. Hopefully previous pointer-scanning phases have
1621 * already completed many of the small buckets [ss, j], so we don't
1622 * have to sort them at all.
1623 */
1624 for (int j = 0; j <= 255; j++) {
1625 final int sb = (ss << 8) + j;
1626 final int ftab_sb = ftab[sb];
1627 if ((ftab_sb & SETMASK) != SETMASK) {
1628 final int lo = ftab_sb & CLEARMASK;
1629 final int hi = (ftab[sb + 1] & CLEARMASK) - 1;
1630 if (hi > lo) {
1631 mainQSort3(dataShadow, lo, hi, 2);
1632 if (firstAttemptShadow
1633 && (this.workDone > workLimitShadow)) {
1634 return;
1635 }
1636 }
1637 ftab[sb] = ftab_sb | SETMASK;
1638 }
1639 }
1640
1641 // Step 2:
1642 // Now scan this big bucket so as to synthesise the
1643 // sorted order for small buckets [t, ss] for all t != ss.
1644
1645 for (int j = 0; j <= 255; j++) {
1646 copy[j] = ftab[(j << 8) + ss] & CLEARMASK;
1647 }
1648
1649 for (int j = ftab[ss << 8] & CLEARMASK, hj = (ftab[(ss + 1) << 8] & CLEARMASK); j < hj; j++) {
1650 final int fmap_j = fmap[j];
1651 c1 = block[fmap_j] & 0xff;
1652 if (!bigDone[c1]) {
1653 fmap[copy[c1]] = (fmap_j == 0) ? lastShadow : (fmap_j - 1);
1654 copy[c1]++;
1655 }
1656 }
1657
1658 for (int j = 256; --j >= 0;) {
1659 ftab[(j << 8) + ss] |= SETMASK;
1660 }
1661
1662 // Step 3:
1663 /*
1664 * The ss big bucket is now done. Record this fact, and update the
1665 * quadrant descriptors. Remember to update quadrants in the
1666 * overshoot area too, if necessary. The "if (i < 255)" test merely
1667 * skips this updating for the last bucket processed, since updating
1668 * for the last bucket is pointless.
1669 */
1670 bigDone[ss] = true;
1671
1672 if (i < 255) {
1673 final int bbStart = ftab[ss << 8] & CLEARMASK;
1674 final int bbSize = (ftab[(ss + 1) << 8] & CLEARMASK) - bbStart;
1675 int shifts = 0;
1676
1677 while ((bbSize >> shifts) > 65534) {
1678 shifts++;
1679 }
1680
1681 for (int j = 0; j < bbSize; j++) {
1682 final int a2update = fmap[bbStart + j];
1683 final char qVal = (char) (j >> shifts);
1684 quadrant[a2update] = qVal;
1685 if (a2update < NUM_OVERSHOOT_BYTES) {
1686 quadrant[a2update + lastShadow + 1] = qVal;
1687 }
1688 }
1689 }
1690
1691 }
1692 }
1693
1694 private void randomiseBlock() {
1695 final boolean[] inUse = this.data.inUse;
1696 final byte[] block = this.data.block;
1697 final int lastShadow = this.last;
1698
1699 for (int i = 256; --i >= 0;) {
1700 inUse[i] = false;
1701 }
1702
1703 int rNToGo = 0;
1704 int rTPos = 0;
1705 for (int i = 0, j = 1; i <= lastShadow; i = j, j++) {
1706 if (rNToGo == 0) {
1707 rNToGo = (char) Rand.rNums(rTPos);
1708 if (++rTPos == 512) {
1709 rTPos = 0;
1710 }
1711 }
1712
1713 rNToGo--;
1714 block[j] ^= ((rNToGo == 1) ? 1 : 0);
1715
1716 // handle 16 bit signed numbers
1717 inUse[block[j] & 0xff] = true;
1718 }
1719
1720 this.blockRandomised = true;
1721 }
1722
1723 private void generateMTFValues() {
1724 final int lastShadow = this.last;
1725 final Data dataShadow = this.data;
1726 final boolean[] inUse = dataShadow.inUse;
1727 final byte[] block = dataShadow.block;
1728 final int[] fmap = dataShadow.fmap;
1729 final char[] sfmap = dataShadow.sfmap;
1730 final int[] mtfFreq = dataShadow.mtfFreq;
1731 final byte[] unseqToSeq = dataShadow.unseqToSeq;
1732 final byte[] yy = dataShadow.generateMTFValues_yy;
1733
1734 // make maps
1735 int nInUseShadow = 0;
1736 for (int i = 0; i < 256; i++) {
1737 if (inUse[i]) {
1738 unseqToSeq[i] = (byte) nInUseShadow;
1739 nInUseShadow++;
1740 }
1741 }
1742 this.nInUse = nInUseShadow;
1743
1744 final int eob = nInUseShadow + 1;
1745
1746 for (int i = eob; i >= 0; i--) {
1747 mtfFreq[i] = 0;
1748 }
1749
1750 for (int i = nInUseShadow; --i >= 0;) {
1751 yy[i] = (byte) i;
1752 }
1753
1754 int wr = 0;
1755 int zPend = 0;
1756
1757 for (int i = 0; i <= lastShadow; i++) {
1758 final byte ll_i = unseqToSeq[block[fmap[i]] & 0xff];
1759 byte tmp = yy[0];
1760 int j = 0;
1761
1762 while (ll_i != tmp) {
1763 j++;
1764 byte tmp2 = tmp;
1765 tmp = yy[j];
1766 yy[j] = tmp2;
1767 }
1768 yy[0] = tmp;
1769
1770 if (j == 0) {
1771 zPend++;
1772 } else {
1773 if (zPend > 0) {
1774 zPend--;
1775 while (true) {
1776 if ((zPend & 1) == 0) {
1777 sfmap[wr] = RUNA;
1778 wr++;
1779 mtfFreq[RUNA]++;
1780 } else {
1781 sfmap[wr] = RUNB;
1782 wr++;
1783 mtfFreq[RUNB]++;
1784 }
1785
1786 if (zPend >= 2) {
1787 zPend = (zPend - 2) >> 1;
1788 } else {
1789 break;
1790 }
1791 }
1792 zPend = 0;
1793 }
1794 sfmap[wr] = (char) (j + 1);
1795 wr++;
1796 mtfFreq[j + 1]++;
1797 }
1798 }
1799
1800 if (zPend > 0) {
1801 zPend--;
1802 while (true) {
1803 if ((zPend & 1) == 0) {
1804 sfmap[wr] = RUNA;
1805 wr++;
1806 mtfFreq[RUNA]++;
1807 } else {
1808 sfmap[wr] = RUNB;
1809 wr++;
1810 mtfFreq[RUNB]++;
1811 }
1812
1813 if (zPend >= 2) {
1814 zPend = (zPend - 2) >> 1;
1815 } else {
1816 break;
1817 }
1818 }
1819 }
1820
1821 sfmap[wr] = (char) eob;
1822 mtfFreq[eob]++;
1823 this.nMTF = wr + 1;
1824 }
1825
1826 private static final class Data extends Object {
1827
1828 // with blockSize 900k
1829 final boolean[] inUse = new boolean[256]; // 256 byte
1830 final byte[] unseqToSeq = new byte[256]; // 256 byte
1831 final int[] mtfFreq = new int[MAX_ALPHA_SIZE]; // 1032 byte
1832 final byte[] selector = new byte[MAX_SELECTORS]; // 18002 byte
1833 final byte[] selectorMtf = new byte[MAX_SELECTORS]; // 18002 byte
1834
1835 final byte[] generateMTFValues_yy = new byte[256]; // 256 byte
1836 final byte[][] sendMTFValues_len = new byte[N_GROUPS][MAX_ALPHA_SIZE]; // 1548
1837 // byte
1838 final int[][] sendMTFValues_rfreq = new int[N_GROUPS][MAX_ALPHA_SIZE]; // 6192
1839 // byte
1840 final int[] sendMTFValues_fave = new int[N_GROUPS]; // 24 byte
1841 final short[] sendMTFValues_cost = new short[N_GROUPS]; // 12 byte
1842 final int[][] sendMTFValues_code = new int[N_GROUPS][MAX_ALPHA_SIZE]; // 6192
1843 // byte
1844 final byte[] sendMTFValues2_pos = new byte[N_GROUPS]; // 6 byte
1845 final boolean[] sentMTFValues4_inUse16 = new boolean[16]; // 16 byte
1846
1847 final int[] stack_ll = new int[QSORT_STACK_SIZE]; // 4000 byte
1848 final int[] stack_hh = new int[QSORT_STACK_SIZE]; // 4000 byte
1849 final int[] stack_dd = new int[QSORT_STACK_SIZE]; // 4000 byte
1850
1851 final int[] mainSort_runningOrder = new int[256]; // 1024 byte
1852 final int[] mainSort_copy = new int[256]; // 1024 byte
1853 final boolean[] mainSort_bigDone = new boolean[256]; // 256 byte
1854
1855 final int[] heap = new int[MAX_ALPHA_SIZE + 2]; // 1040 byte
1856 final int[] weight = new int[MAX_ALPHA_SIZE * 2]; // 2064 byte
1857 final int[] parent = new int[MAX_ALPHA_SIZE * 2]; // 2064 byte
1858
1859 final int[] ftab = new int[65537]; // 262148 byte
1860 // ------------
1861 // 333408 byte
1862
1863 final byte[] block; // 900021 byte
1864 final int[] fmap; // 3600000 byte
1865 final char[] sfmap; // 3600000 byte
1866 // ------------
1867 // 8433529 byte
1868 // ============
1869
1870 /**
1871 * Array instance identical to sfmap, both are used only
1872 * temporarily and indepently, so we do not need to allocate
1873 * additional memory.
1874 */
1875 final char[] quadrant;
1876
1877 Data(int blockSize100k) {
1878 super();
1879
1880 final int n = blockSize100k * BZip2Constants.BASEBLOCKSIZE;
1881 this.block = new byte[(n + 1 + NUM_OVERSHOOT_BYTES)];
1882 this.fmap = new int[n];
1883 this.sfmap = new char[2 * n];
1884 this.quadrant = this.sfmap;
1885 }
1886
1887 }
1888
1889 }