summaryrefslogtreecommitdiffstats
path: root/src/parser/zip_archive.cpp
blob: 50d5da5be3f4b57c01b50eefa8c6f75bf9b77bee (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#include <orcus/zip_archive.hpp>
#include <orcus/zip_archive_stream.hpp>
#include <orcus/string_pool.hpp>

#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <unordered_map>
#include <sstream>
#include <string_view>
#include <iomanip>

#include <zlib.h>
#include <zconf.h>

#define ORCUS_DEBUG_ZIP_ARCHIVE 0

namespace orcus {

namespace {

struct zip_file_param
{
    enum compress_method_type { stored = 0, deflated = 8 };

    std::string_view filename;
    compress_method_type compress_method;
    std::size_t offset_file_header;
    std::size_t size_compressed;
    std::size_t size_uncompressed;

    uint16_t version_made_by;
    uint16_t minimum_version_needed;
    uint16_t flags;
    uint16_t last_modified_time;
    uint16_t last_modified_date;

    uint16_t filename_length;
    uint16_t extra_field_length;
    uint16_t comment_length;

    uint16_t disk_id_where_file_starts;
    uint16_t file_attributes_internal;
    uint32_t file_attributes_external;

    uint32_t crc32;
};

class zip_inflater
{
    z_stream m_zlib_cxt;

    zip_inflater(); // disabled
public:
    zip_inflater(std::vector<unsigned char>& raw_buf, std::vector<unsigned char>& dest_buf, const zip_file_param& param)
    {
        memset(&m_zlib_cxt, 0, sizeof(m_zlib_cxt));
        m_zlib_cxt.next_in = static_cast<Bytef*>(&raw_buf[0]);
        m_zlib_cxt.avail_in = param.size_compressed;

        m_zlib_cxt.next_out = static_cast<Bytef*>(&dest_buf[0]);
        m_zlib_cxt.avail_out = param.size_uncompressed;
    }

    ~zip_inflater()
    {
        inflateEnd(&m_zlib_cxt);
    }

    bool init()
    {
        int err = inflateInit2(&m_zlib_cxt, -MAX_WBITS);
        return err == Z_OK;
    }

    bool inflate()
    {
        int err = ::inflate(&m_zlib_cxt, Z_SYNC_FLUSH);
        if (err >= 0 && m_zlib_cxt.msg)
            return false;

        return true;
    }
};

/**
 * Stream doesn't know its size; only its starting offset position within
 * the file stream.
 */
class zip_stream_parser
{
    zip_archive_stream* m_stream;
    size_t m_pos;
    size_t m_pos_internal;

    void read_string_to_buffer(size_t n, std::vector<unsigned char>& buf)
    {
        if (!n)
            throw zip_error("attempt to read string of zero size.");

        m_stream->seek(m_pos+m_pos_internal);
        m_stream->read(&buf[0], n);
        m_pos_internal += n;
    }

public:
    zip_stream_parser() : m_stream(nullptr), m_pos(0), m_pos_internal(0) {}
    zip_stream_parser(zip_archive_stream* stream, size_t pos) : m_stream(stream), m_pos(pos), m_pos_internal(0) {}

    std::string read_string(size_t n)
    {
        std::vector<unsigned char> buf(n+1, '\0');
        read_string_to_buffer(n, buf);
        return std::string(reinterpret_cast<const char*>(&buf[0]));
    }

    std::vector<uint8_t> read_bytes(std::size_t n)
    {
        if (!n)
            throw zip_error("attempt to read string of zero size.");

        std::vector<uint8_t> buf;
        m_stream->seek(m_pos+m_pos_internal);
        m_stream->read(buf.data(), n);
        m_pos_internal += n;
        return buf;
    }

    std::string_view read_string(size_t n, string_pool& pool)
    {
        std::vector<unsigned char> buf(n+1, '\0');
        read_string_to_buffer(n, buf);
        return pool.intern({reinterpret_cast<const char*>(buf.data()), n}).first;
    }

    void skip_bytes(size_t n)
    {
        m_pos_internal += n;
    }

    uint32_t read_4bytes()
    {
        m_stream->seek(m_pos+m_pos_internal);
        unsigned char buf[4];
        m_stream->read(&buf[0], 4);
        m_pos_internal += 4;

        uint32_t ret = buf[0];
        ret |= (buf[1] << 8);
        ret |= (buf[2] << 16);
        ret |= (buf[3] << 24);

        return ret;
    }

    uint16_t read_2bytes()
    {
        m_stream->seek(m_pos+m_pos_internal);
        unsigned char buf[2];
        m_stream->read(&buf[0], 2);
        m_pos_internal += 2;

        uint16_t ret = buf[0];
        ret |= (buf[1] << 8);

        return ret;
    }

    size_t tell() const
    {
        return m_pos + m_pos_internal;
    }
};

/**
 * Content of the end part of the central directory.
 */
struct central_dir_end
{
    uint32_t magic_number;
    uint16_t this_disk_id;
    uint16_t central_dir_disk_id;
    uint16_t num_central_dir_records_local;
    uint16_t num_celtral_dir_records_total;
    uint32_t size_central_dir;
    size_t central_dir_pos;
    uint16_t comment_length;
};

} // anonymous namespace


zip_file_entry_header::zip_file_entry_header() = default;
zip_file_entry_header::zip_file_entry_header(const zip_file_entry_header& other) = default;
zip_file_entry_header::zip_file_entry_header(zip_file_entry_header&& other) = default;
zip_file_entry_header::~zip_file_entry_header() = default;

zip_file_entry_header& zip_file_entry_header::operator=(const zip_file_entry_header& other) = default;
zip_file_entry_header& zip_file_entry_header::operator=(zip_file_entry_header&& other) = default;

std::ostream& operator<<(std::ostream& os, const zip_file_entry_header& header)
{
    os << "header signature: 0x" << std::hex << std::setfill('0') << std::setw(8) << header.header_signature << "\n"
       << "version needed to extract: " << header.required_version << "\n"
       << "general purpose bit flag: 0x" << std::hex << std::setfill('0') << std::setw(4) << header.flag << "\n"
       << "compression method: " << header.compression_method << "\n"
       << "last modified time: " << header.last_modified_time << "\n"
       << "last modified date: " << header.last_modified_date << "\n"
       << "crc32: 0x" << std::hex << std::setfill('0') << std::setw(8) << header.crc32 << "\n"
       << "compressed size: " << header.compressed_size << "\n"
       << "uncompressed size: " << header.uncompressed_size << "\n"
       << "filename: " << header.filename << "\n"
       << "extra field length: " << header.extra_field.size();

    return os;
}

class zip_archive::impl
{
    typedef std::vector<zip_file_param> file_params_type;
    typedef std::unordered_map<std::string_view, std::size_t> filename_map_type;

    string_pool m_pool;
    zip_archive_stream* m_stream;
    off_t m_stream_size;
    size_t m_central_dir_pos;

    zip_stream_parser m_central_dir_end;

    file_params_type m_file_params;
    filename_map_type m_filenames;

public:
    impl(zip_archive_stream* stream);

    void load();
    zip_file_entry_header get_file_entry_header(std::size_t index) const;
    zip_file_entry_header get_file_entry_header(std::string_view name) const;
    std::string_view get_file_entry_name(size_t pos) const;

    size_t get_file_entry_count() const
    {
        return m_file_params.size();
    }

    std::vector<unsigned char> read_file_entry(std::string_view entry_name) const;

private:

    /**
     * Find the central directory of a zip file, located toward the end before
     * the global comment, and starts with the byte sequence of 0x504b0506.
     */
    size_t seek_central_dir();

    void read_central_dir_end();
    void read_file_entries();
};

zip_archive::impl::impl(zip_archive_stream* stream) :
    m_stream(stream), m_stream_size(0), m_central_dir_pos(0)
{
    if (!m_stream)
        throw zip_error("null stream is not allowed.");

    m_stream_size = m_stream->size();
}

void zip_archive::impl::load()
{
    size_t central_dir_end_pos = seek_central_dir();
    if (!central_dir_end_pos)
        throw zip_error("failed to seek the end position of the central directory");

    m_central_dir_end = zip_stream_parser(m_stream, central_dir_end_pos);

    // Read the end part of the central directory.
    read_central_dir_end();

    // Read file entries that are in the front part of the central directory.
    read_file_entries();
}

zip_file_entry_header zip_archive::impl::get_file_entry_header(std::size_t index) const
{
    if (index >= m_file_params.size())
        throw zip_error("invalid file entry index.");

    const zip_file_param& param = m_file_params[index];
    zip_stream_parser file_header(m_stream, param.offset_file_header);

    zip_file_entry_header header;

    header.header_signature = file_header.read_4bytes();
    header.required_version = file_header.read_2bytes();
    header.flag = file_header.read_2bytes();
    header.compression_method = file_header.read_2bytes();
    header.last_modified_time = file_header.read_2bytes();
    header.last_modified_date = file_header.read_2bytes();
    header.crc32 = file_header.read_4bytes();
    header.compressed_size = file_header.read_4bytes();
    header.uncompressed_size = file_header.read_4bytes();
    uint16_t filename_len = file_header.read_2bytes();
    uint16_t extra_field_len = file_header.read_2bytes();

    if (filename_len)
        header.filename = file_header.read_string(filename_len);

    if (extra_field_len)
        header.extra_field = file_header.read_bytes(extra_field_len);

    return header;
}

zip_file_entry_header zip_archive::impl::get_file_entry_header(std::string_view name) const
{
    auto it = m_filenames.find(name);
    if (it == m_filenames.end())
    {
        std::ostringstream os;
        os << "file entry named '" << name << "' not found";
        throw zip_error(os.str());
    }

    return get_file_entry_header(it->second);
}

void zip_archive::impl::read_file_entries()
{
    m_file_params.clear();

    zip_stream_parser central_dir(m_stream, m_central_dir_pos);
    uint32_t magic_num = central_dir.read_4bytes();

    while (magic_num == 0x02014b50)
    {
        zip_file_param param;

        param.version_made_by = central_dir.read_2bytes();
        param.minimum_version_needed = central_dir.read_2bytes();
        param.flags = central_dir.read_2bytes();
        param.compress_method =
            static_cast<zip_file_param::compress_method_type>(central_dir.read_2bytes());

        param.last_modified_time = central_dir.read_2bytes();
        param.last_modified_date = central_dir.read_2bytes();
        param.crc32 = central_dir.read_4bytes();
        param.size_compressed = central_dir.read_4bytes();
        param.size_uncompressed = central_dir.read_4bytes();
        param.filename_length = central_dir.read_2bytes();
        param.extra_field_length = central_dir.read_2bytes();
        param.comment_length = central_dir.read_2bytes();
        param.disk_id_where_file_starts = central_dir.read_2bytes();
        param.file_attributes_internal = central_dir.read_2bytes();
        param.file_attributes_external = central_dir.read_4bytes();
        param.offset_file_header = central_dir.read_4bytes();

        if (param.filename_length)
            param.filename = central_dir.read_string(param.filename_length, m_pool);

        if (param.extra_field_length)
            // Ignore extra field for now.
            central_dir.skip_bytes(param.extra_field_length);

        if (param.comment_length)
            // Ignore file comment for now.
            central_dir.skip_bytes(param.comment_length);

        magic_num = central_dir.read_4bytes(); // magic number for the next entry.

        m_file_params.push_back(param);
        m_filenames.insert(filename_map_type::value_type(param.filename, m_file_params.size()-1));

#if ORCUS_DEBUG_ZIP_ARCHIVE
        std::cout << "-- file entries" << std::endl;
        printf( "  magic number: 0x%8.8x\n", magic_num);
        std::cout << "  version made by: " << param.version_made_by << std::endl;
        std::cout << "  minimum version needed to extract: " << param.minimum_version_needed << std::endl;
        printf( "  general purpose bit flag: 0x%4.4x\n", param.flags);
        std::cout << "  compression method: " << param.compress_method << " (0=stored, 8=deflated)" << std::endl;
        std::cout << "  file last modified time: " << param.last_modified_time << std::endl;
        std::cout << "  file last modified date: " << param.last_modified_date << std::endl;
        printf( "  crc32: 0x%8.8x\n", param.crc32);
        std::cout << "  compressed size: " << param.size_compressed << std::endl;
        std::cout << "  uncompressed size: " << param.size_uncompressed << std::endl;
        std::cout << "  file name length: " << param.filename_length << std::endl;
        std::cout << "  extra field length: " << param.extra_field_length << std::endl;
        std::cout << "  file comment length: " << param.comment_length << std::endl;
        std::cout << "  disk number where file starts: " << param.disk_id_where_file_starts << std::endl;
        printf( "  internal file attributes: 0x%4.4x\n", param.file_attributes_internal);
        printf( "  external file attributes: 0x%8.8x\n", param.file_attributes_external);
        std::cout << "  relative offset of local file header: " << param.offset_file_header << std::endl;

        if (param.filename_length)
            std::cout << "  filename: '" << param.filename << "'" << std::endl;

        std::cout << "--" << std::endl;
#endif
    }
}

std::string_view zip_archive::impl::get_file_entry_name(std::size_t pos) const
{
    if (pos >= m_file_params.size())
        return std::string_view{};

    return m_file_params[pos].filename;
}

std::vector<unsigned char> zip_archive::impl::read_file_entry(std::string_view entry_name) const
{
    filename_map_type::const_iterator it = m_filenames.find(entry_name);
    if (it == m_filenames.end())
    {
        std::ostringstream os;
        os << "entry named '" << entry_name << "' not found";
        throw zip_error(os.str());
    }


    size_t index = it->second;
    if (index >= m_file_params.size())
        throw zip_error("entry index is out-of-bound");

    const zip_file_param& param = m_file_params[index];

    // Skip the file header section.
    zip_stream_parser file_header(m_stream, param.offset_file_header);
    file_header.skip_bytes(4);
    file_header.skip_bytes(2);
    file_header.skip_bytes(2);
    file_header.skip_bytes(2);
    file_header.skip_bytes(2);
    file_header.skip_bytes(2);
    file_header.skip_bytes(4);
    file_header.skip_bytes(4);
    file_header.skip_bytes(4);
    uint16_t filename_len = file_header.read_2bytes();
    uint16_t extra_field_len = file_header.read_2bytes();
    file_header.skip_bytes(filename_len);
    file_header.skip_bytes(extra_field_len);

    // Data section is immediately followed by the header section.
    m_stream->seek(file_header.tell());

    std::vector<unsigned char> raw_buf(param.size_compressed+1, 0);
    m_stream->read(raw_buf.data(), param.size_compressed);

    switch (param.compress_method)
    {
        case zip_file_param::stored:
        {
            // Not compressed at all.
            return raw_buf;
        }
        case zip_file_param::deflated:
        {
            // deflate compression
            std::vector<unsigned char> zip_buf(param.size_uncompressed+1, 0); // null-terminated
            zip_inflater inflater(raw_buf, zip_buf, param);
            if (!inflater.init())
                throw zip_error("error during initialization of inflater");

            if (!inflater.inflate())
                throw zip_error("error during inflate.");

            return zip_buf;
        }
    }

    throw std::logic_error("compress method can be either 'stored' or 'deflated', but neither has happened");
}

size_t zip_archive::impl::seek_central_dir()
{
    // Search for the position of 0x06054b50 (read in little endian order - so
    // it's 0x50, 0x4b, 0x05, 0x06 in this order) somewhere near the end of
    // the stream.

    unsigned char magic[] = { 0x06, 0x05, 0x4b, 0x50 };
    size_t n_magic = 4;

    off_t max_comment_size = 0xffff;

    size_t buf_size = 22 + max_comment_size; // central directory size is 22 + n (n maxing at 0xffff).
    std::vector<unsigned char> buf(buf_size);

    // Read stream backward and try to find the magic number.

    size_t read_end_pos = m_stream_size;
    while (read_end_pos)
    {
        if (read_end_pos < buf.size())
            // Last segment to read.
            buf.resize(read_end_pos);

        size_t read_pos = read_end_pos - buf.size();
        m_stream->seek(read_pos);
        m_stream->read(&buf[0], buf.size());

        // Search this byte segment for the magic number.
        std::vector<unsigned char>::reverse_iterator i = buf.rbegin(), ie = buf.rend();
        size_t magic_pos = 0;
        for (; i != ie; ++i)
        {
            // 06 05 4b 50
            if (*i == magic[magic_pos])
            {
                ++magic_pos;
                if (magic_pos == n_magic)
                {
                    // magic number is found.
                    size_t dist = distance(buf.rbegin(), i) + 1;
                    size_t pos = read_end_pos - dist;
                    return pos;
                }
            }
            else
                magic_pos = 0;
        }

        read_end_pos -= buf.size();
    }

    return 0;
}

void zip_archive::impl::read_central_dir_end()
{
    central_dir_end content;
    content.magic_number = m_central_dir_end.read_4bytes();
    content.this_disk_id = m_central_dir_end.read_2bytes();
    content.central_dir_disk_id = m_central_dir_end.read_2bytes();
    content.num_central_dir_records_local = m_central_dir_end.read_2bytes();
    content.num_celtral_dir_records_total = m_central_dir_end.read_2bytes();
    content.size_central_dir = m_central_dir_end.read_4bytes();
    content.central_dir_pos = m_central_dir_end.read_4bytes();
    m_central_dir_pos = content.central_dir_pos;

    content.comment_length = m_central_dir_end.read_2bytes();

#if ORCUS_DEBUG_ZIP_ARCHIVE
    std::cout << "-- central directory content" << std::endl;
    printf("  magic number: 0x%8.8x\n", content.magic_number);
    std::cout << "  number of this disk: " << content.this_disk_id << std::endl;
    std::cout << "  disk where central directory starts: " << content.central_dir_disk_id << std::endl;
    std::cout << "  number of central directory records on this disk: " << content.num_central_dir_records_local << std::endl;
    std::cout << "  total number of central directory records: " << content.num_celtral_dir_records_total << std::endl;
    std::cout << "  size of central directory: " << content.size_central_dir << std::endl;
    std::cout << "  offset of start of central directory, relative to start of archive: " << content.central_dir_pos << std::endl;
    std::cout << "  comment length: " << content.comment_length << std::endl;
    std::cout << "--" << std::endl;
#endif
}

zip_archive::zip_archive(zip_archive_stream* stream) : mp_impl(std::make_unique<impl>(stream))
{
}

zip_archive::~zip_archive() = default;

void zip_archive::load()
{
    mp_impl->load();
}

zip_file_entry_header zip_archive::get_file_entry_header(std::size_t index) const
{
    return mp_impl->get_file_entry_header(index);
}

zip_file_entry_header zip_archive::get_file_entry_header(std::string_view name) const
{
    return mp_impl->get_file_entry_header(name);
}

std::string_view zip_archive::get_file_entry_name(std::size_t index) const
{
    return mp_impl->get_file_entry_name(index);
}

size_t zip_archive::get_file_entry_count() const
{
    return mp_impl->get_file_entry_count();
}

std::vector<unsigned char> zip_archive::read_file_entry(std::string_view entry_name) const
{
    return mp_impl->read_file_entry(entry_name);
}

}
/* vim:set shiftwidth=4 softtabstop=4 expandtab: */