001/*
002 *  Licensed to the Apache Software Foundation (ASF) under one or more
003 *  contributor license agreements.  See the NOTICE file distributed with
004 *  this work for additional information regarding copyright ownership.
005 *  The ASF licenses this file to You under the Apache License, Version 2.0
006 *  (the "License"); you may not use this file except in compliance with
007 *  the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 *  Unless required by applicable law or agreed to in writing, software
012 *  distributed under the License is distributed on an "AS IS" BASIS,
013 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 *  See the License for the specific language governing permissions and
015 *  limitations under the License.
016 *
017 */
018package org.apache.commons.compress.archivers.zip;
019
020
021import java.io.Closeable;
022import java.io.File;
023import java.io.FileNotFoundException;
024import java.io.IOException;
025import java.io.InputStream;
026import java.nio.file.Path;
027import java.util.Iterator;
028import java.util.Queue;
029import java.util.concurrent.ConcurrentLinkedQueue;
030import java.util.concurrent.atomic.AtomicBoolean;
031import java.util.zip.Deflater;
032
033import org.apache.commons.compress.parallel.FileBasedScatterGatherBackingStore;
034import org.apache.commons.compress.parallel.ScatterGatherBackingStore;
035import org.apache.commons.compress.utils.BoundedInputStream;
036
037/**
038 * A zip output stream that is optimized for multi-threaded scatter/gather construction of zip files.
039 * <p>
040 * The internal data format of the entries used by this class are entirely private to this class
041 * and are not part of any public api whatsoever.
042 * </p>
043 * <p>It is possible to extend this class to support different kinds of backing storage, the default
044 * implementation only supports file-based backing.
045 * </p>
046 * Thread safety: This class supports multiple threads. But the "writeTo" method must be called
047 * by the thread that originally created the {@link ZipArchiveEntry}.
048 *
049 * @since 1.10
050 */
051public class ScatterZipOutputStream implements Closeable {
052    private final Queue<CompressedEntry> items = new ConcurrentLinkedQueue<>();
053    private final ScatterGatherBackingStore backingStore;
054    private final StreamCompressor streamCompressor;
055    private final AtomicBoolean isClosed = new AtomicBoolean();
056    private ZipEntryWriter zipEntryWriter;
057
058    private static class CompressedEntry {
059        final ZipArchiveEntryRequest zipArchiveEntryRequest;
060        final long crc;
061        final long compressedSize;
062        final long size;
063
064        public CompressedEntry(final ZipArchiveEntryRequest zipArchiveEntryRequest, final long crc, final long compressedSize, final long size) {
065            this.zipArchiveEntryRequest = zipArchiveEntryRequest;
066            this.crc = crc;
067            this.compressedSize = compressedSize;
068            this.size = size;
069        }
070
071        /**
072         * Update the original {@link ZipArchiveEntry} with sizes/crc
073         * Do not use this methods from threads that did not create the instance itself !
074         * @return the zipArchiveEntry that is basis for this request
075         */
076
077        public ZipArchiveEntry transferToArchiveEntry(){
078            final ZipArchiveEntry entry = zipArchiveEntryRequest.getZipArchiveEntry();
079            entry.setCompressedSize(compressedSize);
080            entry.setSize(size);
081            entry.setCrc(crc);
082            entry.setMethod(zipArchiveEntryRequest.getMethod());
083            return entry;
084        }
085    }
086
087    public ScatterZipOutputStream(final ScatterGatherBackingStore backingStore,
088                                  final StreamCompressor streamCompressor) {
089        this.backingStore = backingStore;
090        this.streamCompressor = streamCompressor;
091    }
092
093    /**
094     * Add an archive entry to this scatter stream.
095     *
096     * @param zipArchiveEntryRequest The entry to write.
097     * @throws IOException    If writing fails
098     */
099    public void addArchiveEntry(final ZipArchiveEntryRequest zipArchiveEntryRequest) throws IOException {
100        try (final InputStream payloadStream = zipArchiveEntryRequest.getPayloadStream()) {
101            streamCompressor.deflate(payloadStream, zipArchiveEntryRequest.getMethod());
102        }
103        items.add(new CompressedEntry(zipArchiveEntryRequest, streamCompressor.getCrc32(),
104                                      streamCompressor.getBytesWrittenForLastEntry(), streamCompressor.getBytesRead()));
105    }
106
107    /**
108     * Write the contents of this scatter stream to a target archive.
109     *
110     * @param target The archive to receive the contents of this {@link ScatterZipOutputStream}.
111     * @throws IOException If writing fails
112     * @see #zipEntryWriter()
113     */
114    public void writeTo(final ZipArchiveOutputStream target) throws IOException {
115        backingStore.closeForWriting();
116        try (final InputStream data = backingStore.getInputStream()) {
117            for (final CompressedEntry compressedEntry : items) {
118                try (final BoundedInputStream rawStream = new BoundedInputStream(data,
119                        compressedEntry.compressedSize)) {
120                    target.addRawArchiveEntry(compressedEntry.transferToArchiveEntry(), rawStream);
121                }
122            }
123        }
124    }
125
126    public static class ZipEntryWriter implements Closeable {
127        private final Iterator<CompressedEntry> itemsIterator;
128        private final InputStream itemsIteratorData;
129
130        public ZipEntryWriter(final ScatterZipOutputStream scatter) throws IOException {
131            scatter.backingStore.closeForWriting();
132            itemsIterator = scatter.items.iterator();
133            itemsIteratorData = scatter.backingStore.getInputStream();
134        }
135
136        @Override
137        public void close() throws IOException {
138            if (itemsIteratorData != null) {
139                itemsIteratorData.close();
140            }
141        }
142
143        public void writeNextZipEntry(final ZipArchiveOutputStream target) throws IOException {
144            final CompressedEntry compressedEntry = itemsIterator.next();
145            try (final BoundedInputStream rawStream = new BoundedInputStream(itemsIteratorData, compressedEntry.compressedSize)) {
146                target.addRawArchiveEntry(compressedEntry.transferToArchiveEntry(), rawStream);
147            }
148        }
149    }
150
151    /**
152     * Get a zip entry writer for this scatter stream.
153     * @throws IOException If getting scatter stream input stream
154     * @return the ZipEntryWriter created on first call of the method
155     */
156    public ZipEntryWriter zipEntryWriter() throws IOException {
157        if (zipEntryWriter == null) {
158            zipEntryWriter = new ZipEntryWriter(this);
159        }
160        return zipEntryWriter;
161    }
162
163    /**
164     * Closes this stream, freeing all resources involved in the creation of this stream.
165     * @throws IOException If closing fails
166     */
167    @Override
168    public void close() throws IOException {
169        if (!isClosed.compareAndSet(false, true)) {
170            return;
171        }
172        try {
173            if (zipEntryWriter != null) {
174                zipEntryWriter.close();
175            }
176            backingStore.close();
177        } finally {
178            streamCompressor.close();
179        }
180    }
181
182    /**
183     * Create a {@link ScatterZipOutputStream} with default compression level that is backed by a file
184     *
185     * @param file The file to offload compressed data into.
186     * @return A ScatterZipOutputStream that is ready for use.
187     * @throws FileNotFoundException if the file cannot be found
188     */
189    public static ScatterZipOutputStream fileBased(final File file) throws FileNotFoundException {
190        return pathBased(file.toPath(), Deflater.DEFAULT_COMPRESSION);
191    }
192
193    /**
194     * Create a {@link ScatterZipOutputStream} with default compression level that is backed by a file
195     * @param path The path to offload compressed data into.
196     * @return A ScatterZipOutputStream that is ready for use.
197     * @throws FileNotFoundException if the path cannot be found
198     * @since 1.22
199     */
200    public static ScatterZipOutputStream pathBased(final Path path) throws FileNotFoundException {
201        return pathBased(path, Deflater.DEFAULT_COMPRESSION);
202    }
203
204    /**
205     * Create a {@link ScatterZipOutputStream} that is backed by a file
206     *
207     * @param file             The file to offload compressed data into.
208     * @param compressionLevel The compression level to use, @see #Deflater
209     * @return A  ScatterZipOutputStream that is ready for use.
210     * @throws FileNotFoundException if the file cannot be found
211     */
212    public static ScatterZipOutputStream fileBased(final File file, final int compressionLevel) throws FileNotFoundException {
213        return pathBased(file.toPath(), compressionLevel);
214    }
215
216    /**
217     * Create a {@link ScatterZipOutputStream} that is backed by a file
218     * @param path The path to offload compressed data into.
219     * @param compressionLevel The compression level to use, @see #Deflater
220     * @return A ScatterZipOutputStream that is ready for use.
221     * @throws FileNotFoundException if the path cannot be found
222     * @since 1.22
223     */
224    public static ScatterZipOutputStream pathBased(final Path path, final int compressionLevel) throws FileNotFoundException {
225        final ScatterGatherBackingStore bs = new FileBasedScatterGatherBackingStore(path);
226        // lifecycle is bound to the ScatterZipOutputStream returned
227        final StreamCompressor sc = StreamCompressor.create(compressionLevel, bs); //NOSONAR
228        return new ScatterZipOutputStream(bs, sc);
229    }
230}